From 4de586089c9d8093f0f5d3a9194b7495e9ab2325 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Fri, 19 Jun 2026 19:17:17 -0400 Subject: [PATCH] Chez Phase 3 inc 5a: port reader atom layer to jolt.reader (Clojure) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starts taking the reader off Janet (src/jolt/reader.janet, 831 lines) into portable jolt-core Clojure. jolt.reader holds the lexing/parsing LOGIC; form construction + string->number parsing delegate to the jolt.host contract — a Clojure source file can't write a {:jolt/type :symbol} literal (parses as a tagged form) and the concrete representation is the host's to own. Same split the analyzer/emitter already use. Once cross-compiled this runs on Chez so compile- from-source needs no Janet reader. inc 5a = the atom layer: whitespace/comments, symbols (+ nil/true/false), keywords, strings (escapes), numbers (sign/hex/radix/ratio/fractional/exponent, trailing N/M), characters. Collections, quote/deref/meta and dispatch (#) follow in 5b/5c (throw not-yet-ported). Positions are char indices (Janet uses bytes); identical for ASCII and the gate compares form VALUES, not positions. host_iface.janet gains four reader primitives on the contract: form-make-symbol, form-make-char, form-char-from-name, form-scan-number (the irreducible host bits the portable reader rests on). Additive — new jolt.host interns, nothing else changed. Surfaced jolt-if19 (Janet seed reader: +N literals error instead of reading as N; read-number strips only the - sign). The port reproduces it; both-throw counts as faithful parity in the gate. Gate: reader-parity 64/64 (symbols/keywords/strings/ints/hex/radix/ratio/floats/ exponent/N-M/chars). Full jpm gate green after clean rebuild, conformance 355x3. jolt-50xx. --- jolt-core/jolt/reader.clj | 195 ++++++++++++++++++++++++++++++++++ src/jolt/host_iface.janet | 16 +++ test/chez/reader-parity.janet | 66 ++++++++++++ 3 files changed, 277 insertions(+) create mode 100644 jolt-core/jolt/reader.clj create mode 100644 test/chez/reader-parity.janet diff --git a/jolt-core/jolt/reader.clj b/jolt-core/jolt/reader.clj new file mode 100644 index 0000000..1b3c5bf --- /dev/null +++ b/jolt-core/jolt/reader.clj @@ -0,0 +1,195 @@ +(ns jolt.reader + "Portable Clojure reader: source text -> reader forms (Chez Phase 3, jolt-cf1q.4). + + The no-Janet replacement for src/jolt/reader.janet. All the lexing/parsing LOGIC + is portable Clojure; form CONSTRUCTION and string->number parsing delegate to the + jolt.host contract (form-make-symbol/char, form-char-from-name, form-scan-number) + — a Clojure source file cannot write a {:jolt/type :symbol} literal (it parses as + a tagged reader form), and the concrete form representation is the host's to own. + Same split the analyzer uses for the form-* readers. Once cross-compiled this runs + ON Chez so compile-from-source needs no Janet reader. + + Positions are CHARACTER indices (the Janet reader uses byte indices); for ASCII + source they coincide, and form VALUES are identical either way — the parity gate + compares values, not positions. + + INCREMENT 5a (jolt-50xx): the ATOM layer — whitespace/comments, symbols (+ nil/ + true/false), keywords, strings, numbers (sign/hex/radix/ratio/fractional/ + exponent, trailing N/M), characters. Collections, quote/deref/meta, and dispatch + (#) land in 5b/5c (they throw not-yet-ported so a hit is loud)." + (:require [clojure.string :as str] + [jolt.host :refer [form-make-symbol form-make-char form-char-from-name + form-scan-number]])) + +;; Source access by CHARACTER codepoint, mirroring the Janet reader's byte access +;; (identical for ASCII). cp = codepoint at i; len = character count. +(defn- cp [s i] (int (nth s i))) +(defn- len [s] (count s)) + +(defn- whitespace? [c] (or (= c 32) (= c 9) (= c 10) (= c 13) (= c 44))) ; space tab nl cr , +(defn- digit? [c] (and (>= c 48) (<= c 57))) +(defn- hex-digit? [c] + (or (digit? c) (and (>= c 65) (<= c 70)) (and (>= c 97) (<= c 102)))) +(defn- symbol-start? [c] + (or (and (>= c 65) (<= c 90)) (and (>= c 97) (<= c 122)) + (= c 42) (= c 43) (= c 33) (= c 95) (= c 45) (= c 63) (= c 46) + (= c 60) (= c 62) (= c 61) (= c 38) (= c 124) (= c 36) (= c 37) (= c 47))) +(defn- symbol-char? [c] + (or (symbol-start? c) (digit? c) (= c 35) (= c 39) (= c 58))) ; + # ' : + +(defn- skip-whitespace [s pos] + (if (and (< pos (len s)) (whitespace? (cp s pos))) + (recur s (inc pos)) + pos)) + +(defn- read-until-newline [s pos] + (if (or (>= pos (len s)) (= (cp s pos) 10)) pos (recur s (inc pos)))) + +;; --- symbols ----------------------------------------------------------------- +(defn- read-symbol-name [s pos end] + (if (and (< end (len s)) (symbol-char? (cp s end))) (recur s pos (inc end)) end)) + +(defn- read-symbol* [s pos] + (let [end (read-symbol-name s pos pos)] + (when (= end pos) + (throw (ex-info (str "Unrecognized character: " (char (cp s pos))) {}))) + (let [nm (subs s pos end)] + (cond + (= nm "nil") [nil end] + (= nm "true") [true end] + (= nm "false") [false end] + :else [(form-make-symbol nm) end])))) + +;; --- keywords ---------------------------------------------------------------- +(defn- read-keyword-name [s pos end] + (if (and (< end (len s)) (symbol-char? (cp s end))) (recur s pos (inc end)) end)) + +(defn- read-keyword* [s pos] + ;; pos is at the first colon; ::foo is treated as :foo (no auto-resolution), + ;; matching the Janet reader. + (let [start (if (and (< (inc pos) (len s)) (= (cp s (inc pos)) 58)) (+ pos 2) (inc pos)) + end (read-keyword-name s start start)] + [(keyword (subs s start end)) end])) + +;; --- strings ----------------------------------------------------------------- +(defn- escape-char [c] + (cond (= c 110) "\n" (= c 116) "\t" (= c 114) "\r" (= c 92) "\\" (= c 34) "\"" + :else (str (char c)))) + +(defn- read-string* [s pos] + ;; pos at opening double-quote + (loop [p (inc pos) acc []] + (when (>= p (len s)) (throw (ex-info "Unterminated string" {}))) + (let [c (cp s p)] + (cond + (= c 92) (let [np (inc p)] + (when (>= np (len s)) (throw (ex-info "Unterminated escape" {}))) + (recur (+ p 2) (conj acc (escape-char (cp s np))))) + (= c 34) [(apply str acc) (inc p)] + :else (recur (inc p) (conj acc (str (char c)))))))) + +;; --- numbers ----------------------------------------------------------------- +(defn- read-digits [s pos end] + (if (and (< end (len s)) (digit? (cp s end))) (recur s pos (inc end)) end)) +(defn- read-hex-digits [s pos end] + (if (and (< end (len s)) (hex-digit? (cp s end))) (recur s pos (inc end)) end)) + +;; Value of an alphanumeric digit for radix parsing (0-9, a-z/A-Z = 10-35). +(defn- radix-digit-val [c] + (cond + (and (>= c 48) (<= c 57)) (- c 48) + (and (>= c 97) (<= c 122)) (+ 10 (- c 97)) + (and (>= c 65) (<= c 90)) (+ 10 (- c 65)) + :else nil)) +(defn- read-alnum [s pos end] + (if (and (< end (len s)) (radix-digit-val (cp s end))) (recur s pos (inc end)) end)) + +(defn- read-exponent [s end] + ;; if s[end] is e/E (optionally signed) followed by digits, return index past it + (if (and (< end (len s)) (let [c (cp s end)] (or (= c 101) (= c 69)))) + (let [p (if (and (< (inc end) (len s)) (let [c (cp s (inc end))] (or (= c 43) (= c 45)))) + (+ end 2) (inc end)) + de (read-digits s p p)] + (if (> de p) de end)) + end)) + +;; Jolt has no bignum/ratio: N (bigint) / M (bigdec) suffixes read as the plain +;; number, a ratio a/b reads as the double quotient, radixed ints by base. +(defn- read-number* [s pos] + (let [length (len s) + neg (and (< pos length) (= (cp s pos) 45)) + start (if neg (inc pos) pos) + hex? (and (< (inc start) length) (= (cp s start) 48) + (let [c1 (cp s (inc start))] (or (= c1 120) (= c1 88))))] ; 0x / 0X + (if hex? + (let [hs (+ start 2) he (read-hex-digits s hs hs)] + (when (= he hs) (throw (ex-info "Expected hex digits" {}))) + (let [he2 (if (and (< he length) (= (cp s he) 78)) (inc he) he) ; trailing N + val (form-scan-number (str "0x" (subs s hs he)))] + [(if neg (- val) val) he2])) + (let [iend (read-digits s start start)] + (when (= iend start) (throw (ex-info "Expected number" {}))) + (cond + ;; radix integer r + (and (< iend length) (let [c (cp s iend)] (or (= c 114) (= c 82)))) + (let [base (form-scan-number (subs s start iend)) + ds (inc iend) de (read-alnum s ds ds)] + (when (= de ds) (throw (ex-info "Expected radix digits" {}))) + (let [acc (reduce (fn [a i] (+ (* a base) (radix-digit-val (cp s i)))) 0 (range ds de))] + [(if neg (- acc) acc) de])) + ;; ratio / (only when a digit follows the slash) + (and (< (inc iend) length) (= (cp s iend) 47) (digit? (cp s (inc iend)))) + (let [ds (inc iend) de (read-digits s ds ds) + numr (form-scan-number (subs s start iend)) + den (form-scan-number (subs s ds de))] + [(if neg (- (/ numr den)) (/ numr den)) de]) + ;; fractional and/or exponent, optional trailing N/M + :else + (let [frac-end (if (and (< iend length) (= (cp s iend) 46)) + (let [fs (inc iend) fe (read-digits s fs fs)] + (when (= fe fs) (throw (ex-info "Expected digit after ." {}))) + fe) + iend) + exp-end (read-exponent s frac-end) + val (form-scan-number (subs s start exp-end)) + fin (if (and (< exp-end length) (let [c (cp s exp-end)] (or (= c 78) (= c 77)))) + (inc exp-end) exp-end)] + [(if neg (- val) val) fin])))))) + +;; --- characters -------------------------------------------------------------- +(defn- read-char-name-end [s pos] + (if (and (< pos (len s)) (symbol-char? (cp s pos))) (recur s (inc pos)) pos)) + +(defn- read-char* [s pos] + (when (>= (inc pos) (len s)) (throw (ex-info "unexpected end of input after \\" {}))) + (let [end (read-char-name-end s (inc pos))] + (if (= end (inc pos)) + ;; a non-symbol char right after \ is a one-character literal of itself + [(form-make-char (cp s (inc pos))) (+ pos 2)] + [(form-char-from-name (subs s (inc pos) end)) end]))) + +;; --- dispatcher -------------------------------------------------------------- +(defn- number-start? [s pos c] + (or (digit? c) + (and (= c 45) (< (inc pos) (len s)) (digit? (cp s (inc pos)))) + (and (= c 43) (< (inc pos) (len s)) (digit? (cp s (inc pos)))))) + +(defn read-form [s pos] + (let [pos (skip-whitespace s pos)] + (if (>= pos (len s)) + [nil pos] + (let [c (cp s pos)] + (cond + (= c 59) (read-form s (read-until-newline s pos)) ; ; comment: skip + reread + (= c 34) (read-string* s pos) + (= c 58) (read-keyword* s pos) + (= c 92) (read-char* s pos) + (number-start? s pos c) (read-number* s pos) + (symbol-start? c) (read-symbol* s pos) + :else (throw (ex-info (str "read-form: not yet ported (inc 5a atoms): '" + (char c) "' (" c ")") {}))))))) + +(defn read-one + "Read the first form of `s` (skipping leading trivia). Returns the form." + [s] + (first (read-form s 0))) diff --git a/src/jolt/host_iface.janet b/src/jolt/host_iface.janet index 364b3d9..693e402 100644 --- a/src/jolt/host_iface.janet +++ b/src/jolt/host_iface.janet @@ -282,8 +282,24 @@ (defn h-record-shapes [ctx] (or (get (ctx :env) :record-shapes) @{})) +# Reader form CONSTRUCTORS (Chez Phase 3, jolt-cf1q.4). The portable Clojure +# reader (jolt.reader) holds all the LEXING/PARSING logic but must DELEGATE form +# construction to the host — a Clojure source file cannot write a {:jolt/type +# :symbol} literal (it parses as a tagged reader form, CLAUDE.md), and the +# concrete representation (Janet struct/array/tuple here; something else on Chez) +# is the host's to own. Same split the analyzer already uses for form-* readers. +(defn h-make-symbol [name] (rdr/make-symbol name)) +(defn h-make-char [code] (make-char code)) +(defn h-char-from-name [name] (char-from-name name)) +# Parse a numeric literal substring to a number. The reader keeps the radix/ratio/ +# sign/exponent LOGIC portable; only the final string->number bottoms out here +# (Janet scan-number; Chez string->number). Returns nil on a non-number. +(defn h-scan-number [str] (scan-number str)) + (def- exports {"form-sym?" h-sym? "form-sym-name" h-sym-name "form-sym-ns" h-sym-ns + "form-make-symbol" h-make-symbol "form-make-char" h-make-char + "form-char-from-name" h-char-from-name "form-scan-number" h-scan-number "ref-put!" h-ref-put! "ref-get" h-ref-get "tagged-table" h-tagged-table diff --git a/test/chez/reader-parity.janet b/test/chez/reader-parity.janet new file mode 100644 index 0000000..62caa68 --- /dev/null +++ b/test/chez/reader-parity.janet @@ -0,0 +1,66 @@ +# Chez Phase 3 inc 5a (jolt-50xx) — value-parity gate for the PORTABLE Clojure +# reader (jolt.reader) vs the Janet seed reader (src/jolt/reader.janet, oracle). +# +# jolt.reader holds the lexing/parsing LOGIC in portable Clojure and delegates +# form construction + number parsing to the jolt.host contract. Here it runs +# interpreted ON THE JANET HOST (loaded via bootstrap-load-source); each input is +# read by BOTH readers and the resulting FORMS compared with jolt's own = (so the +# representation, host-built either way, matches structurally). Positions differ +# (char vs byte indices) and are not compared. +# +# janet test/chez/reader-parity.janet (from repo root) +(import ../../src/jolt/api :as api) +(import ../../src/jolt/backend :as backend) +(import ../../src/jolt/reader :as r) +(import ../../src/jolt/types_ctx :as tctx) +(import ../../src/jolt/types_ns :as tns) +(import ../../src/jolt/types_var :as tvar) +(import ../../src/jolt/core :as core) + +(var total 0) (var fails 0) +(defn ok [name pred &opt extra] + (++ total) + (if pred (printf "ok: %s" name) + (do (++ fails) (printf "FAIL: %s %s" name (or extra ""))))) + +(def ctx (api/init {:compile? true})) +(def src (get (get (ctx :env) :embedded-sources) "jolt.reader")) +(assert src "jolt.reader not embedded (check stdlib_embed collect)") +(backend/bootstrap-load-source ctx "jolt.reader" src) +(def read-one (tvar/var-get (tns/ns-find (tctx/ctx-find-ns ctx "jolt.reader") "read-one"))) +(assert read-one "jolt.reader/read-one not found") + +# jolt's own value equality (the host =), the same comparator the corpus gate uses. +(defn jeq [a b] (core/jolt-equal? a b)) + +(defn check [input] + (def w (protect (in (r/parse-next input) 0))) # Janet seed reader (oracle) + (def g (protect (read-one input))) # portable Clojure reader + (cond + # both readers throw on the same input = faithful parity (e.g. the +N latent + # bug, jolt-if19 — reader.janet dispatches +digit to read-number but never + # strips the +, so "+5" errors; the port reproduces it). + (and (not (w 0)) (not (g 0))) (ok input true) + (not (w 0)) (ok input false (string "janet threw, clj didn't: clj=" (string/format "%p" (g 1)))) + (not (g 0)) (ok input false (string "clj threw, janet ok: " (string (g 1)))) + (ok input (jeq (w 1) (g 1)) + (string "clj=" (string/format "%p" (g 1)) " janet=" (string/format "%p" (w 1)))))) + +# --- inc 5a: atoms ------------------------------------------------------------- +# nil / bool +(each i ["nil" "true" "false"] (check i)) +# symbols (plain, ns'd, punctuation, special chars) +(each i ["foo" "foo-bar" "my.ns/bar" "+" "-" "*" "->" "<=" "some?" "a1" "x'" "ns/+"] (check i)) +# keywords +(each i [":a" ":foo-bar" ":my.ns/key" "::auto" ":a1" ":+"] (check i)) +# strings (escapes) +(each i [`"hello"` `"with space"` `"tab\there"` `"nl\nhere"` `"q\"q"` `"back\\slash"` `""`] (check i)) +# integers / signs / hex / radix +(each i ["0" "42" "-7" "+5" "123456" "0xFF" "0x10" "-0xff" "2r1010" "16rFF" "36rZ" "8r17"] (check i)) +# floats / exponent / ratio / N|M suffix +(each i ["3.14" "-2.5" "0.0" "1e10" "1.5e-3" "2E5" "10N" "3.14M" "1/2" "-3/4" "22/7"] (check i)) +# characters +(each i [`\a` `\Z` `\0` `\newline` `\tab` `\space` `\return` `\\` `\(` `\{` `\%` `A` `\o101`] (check i)) + +(printf "\n%d/%d ok" (- total fails) total) +(when (> fails 0) (os/exit 1))