From 37c433bd4abf663fbc45118490cb5b22182e44d9 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 17 Jun 2026 19:44:18 -0400 Subject: [PATCH] Chez Phase 1 (increment 3i): regex via vendored irregex Closes the last clojure.core prelude emit gap (parse-uuid): the whole non-macro core now lowers to Scheme (prelude reach 355/355). A #"..." literal analyzes to a :regex IR node. The Chez back end emits a jolt-regex value over irregex (Alex Shinn, BSD), vendored as the vendor/irregex submodule -- a portable Scheme regex with PCRE/Java-style string patterns and first-class Chez support. host/chez/regex.ss wraps jolt's re-* surface over it: irregex-match -> re-matches (anchored), irregex-search -> re-find, groups as Clojure [whole g1 ...] vectors, re-seq as a jolt seq. re-pattern/re-matches/re-find/re-seq/regex? are def-var!'d into clojure.core so prelude / -e code resolves them. They stay OUT of the subset native-ops on purpose: irregex's Unicode/property-class semantics differ from the seed's byte-PEG approximation, so keeping them prelude-only avoids dragging engine-difference divergences into the subset-parity corpus. The Janet back end punts :regex to the interpreter (the seed compiles #"..." to a Janet PEG), so the main language is unchanged. Only two adaptations for Chez's top level: a cond-expand shim (Chez's is library-only) and a normalizing error wrapper (silences irregex's 1-arg error warnings). rt.ss load is ~0.18s. emit-test 131/131 (regex literal + re-* parity vs the CLI oracle); prelude reach 355/355; Chez subset 672/672, 0 divergences; full gate green. --- .gitmodules | 3 ++ host/chez/emit.janet | 5 ++ host/chez/regex.ss | 81 ++++++++++++++++++++++++++++++ host/chez/rt.ss | 6 +++ jolt-core/jolt/analyzer.clj | 5 ++ src/jolt/backend.janet | 3 ++ src/jolt/host_iface.janet | 8 +++ test/chez/README.md | 28 ++++++----- test/chez/core-prelude-probe.janet | 2 +- test/chez/emit-test.janet | 36 +++++++++++++ vendor/irregex | 1 + 11 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 host/chez/regex.ss create mode 160000 vendor/irregex diff --git a/.gitmodules b/.gitmodules index 959062d..0ae0a56 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "vendor/clojure-test-suite"] path = vendor/clojure-test-suite url = https://github.com/jank-lang/clojure-test-suite.git +[submodule "vendor/irregex"] + path = vendor/irregex + url = https://github.com/ashinn/irregex.git diff --git a/host/chez/emit.janet b/host/chez/emit.janet index 7c56fb3..363d2c7 100644 --- a/host/chez/emit.janet +++ b/host/chez/emit.janet @@ -369,6 +369,11 @@ :throw (string "(jolt-throw " (emit (get node :expr)) ")") :try (emit-try node) :quote (emit-quoted (get node :form)) + # regex literal #"…" -> a jolt-regex value (regex.ss compiles the source via + # the vendored irregex). %j quotes+escapes the source; a backslash in the + # pattern becomes \\ in the Scheme string literal -> the 1-char backslash + # irregex expects (same escaping emit-const uses for strings). + :regex (string "(jolt-regex " (string/format "%j" (get node :source)) ")") # host interop (jolt-0kf5): (.method target arg*) -> (jolt-host-call "method" # target arg*). Only the methods the RT dispatcher (rt.ss) actually shims are # IN the subset; any other method is out of subset (a clean emit-time reject, diff --git a/host/chez/regex.ss b/host/chez/regex.ss new file mode 100644 index 0000000..91a173c --- /dev/null +++ b/host/chez/regex.ss @@ -0,0 +1,81 @@ +;; Phase 1 (jolt-cf1q.2) — regex on Chez via vendored irregex (jolt-i0s3). +;; +;; jolt's seed regex (src/jolt/regex.janet) compiles patterns to Janet's PEG +;; engine; Chez has no regex at all. Rather than re-host that engine, we vendor +;; Alex Shinn's irregex (vendor/irregex, BSD) — a portable Scheme regex with +;; PCRE/Java-style STRING patterns — and wrap jolt's re-* surface over it. +;; +;; irregex maps cleanly onto the Clojure fns: irregex-match is an anchored +;; whole-string match (= re-matches), irregex-search finds the first match +;; anywhere (= re-find), irregex-match-substring extracts group N (0 = whole). +;; Results follow Clojure shape: a 0-group match is the whole string; a grouped +;; match is a jolt VECTOR [whole g1 ...] (a non-participating group is nil); a nil +;; result is jolt-nil; re-seq is a jolt seq (nil when there are no matches). +;; +;; The re-* fns are def-var!'d into clojure.core so prelude / -e code resolves +;; them at runtime (they're NOT subset native-ops: irregex's Unicode/property- +;; class semantics differ from the seed's byte-PEG approximation, so they stay out +;; of the subset-parity corpus). Loaded from rt.ss after def-var! is defined. + +;; irregex.scm is portable R[457]RS; two small adaptations for Chez's top level: +;; a cond-expand at expression position (Chez's is library-only), and `error` +;; called with a lone string (Chez's error wants who+msg). The wrapper normalizes +;; both without changing behavior for valid patterns. +(define-syntax cond-expand + (syntax-rules (else) + ((_ (else e ...)) (begin e ...)) + ((_ (else e ...) c ...) (begin e ...)) + ((_ (req e ...) c ...) (cond-expand c ...)) + ((_) (if #f #f)))) +(define %chez-error error) +(define (error . args) + (if (and (pair? args) (string? (car args))) + (apply %chez-error #f args) + (apply %chez-error args))) +(load "vendor/irregex/irregex.scm") + +;; A jolt regex value: the source string (for printing / str) + the compiled +;; irregex. regex? recognizes it; the printer renders #"source". +(define-record-type regex-t (fields source irx) (nongenerative jolt-regex-v1)) +(define (jolt-regex source) (make-regex-t source (irregex source))) +(define (jolt-regex? x) (regex-t? x)) +(define (jolt-re-pattern x) (if (regex-t? x) x (jolt-regex x))) + +;; An irregex match -> the Clojure result: whole string (no groups) or the +;; [whole g1 ... gn] vector (nil for a non-participating group). +(define (irx-result m) + (let ((n (irregex-match-num-submatches m))) + (if (= n 0) + (irregex-match-substring m 0) + (let loop ((i n) (acc '())) + (if (< i 0) + (apply jolt-vector acc) + (let ((s (irregex-match-substring m i))) + (loop (- i 1) (cons (if s s jolt-nil) acc)))))))) + +(define (jolt-re-matches re s) + (let ((m (irregex-match (regex-t-irx (jolt-re-pattern re)) s))) + (if m (irx-result m) jolt-nil))) + +(define (jolt-re-find re s) + (let ((m (irregex-search (regex-t-irx (jolt-re-pattern re)) s))) + (if m (irx-result m) jolt-nil))) + +;; All non-overlapping matches, left to right. Advance past each match end (or by +;; one on a zero-width match). nil when there are no matches (Clojure: seq-able as +;; nil, so (if-let [m (re-seq ...)] ...) works), matching the seed. +(define (jolt-re-seq re s) + (let ((irx (regex-t-irx (jolt-re-pattern re))) + (len (string-length s))) + (let loop ((start 0) (acc '())) + (let ((m (and (<= start len) (irregex-search irx s start)))) + (if m + (let ((e (irregex-match-end-index m 0))) + (loop (if (> e start) e (+ start 1)) (cons (irx-result m) acc))) + (list->cseq (reverse acc))))))) + +(def-var! "clojure.core" "re-pattern" jolt-re-pattern) +(def-var! "clojure.core" "re-matches" jolt-re-matches) +(def-var! "clojure.core" "re-find" jolt-re-find) +(def-var! "clojure.core" "re-seq" jolt-re-seq) +(def-var! "clojure.core" "regex?" jolt-regex?) diff --git a/host/chez/rt.ss b/host/chez/rt.ss index d356961..e5ec6f1 100644 --- a/host/chez/rt.ss +++ b/host/chez/rt.ss @@ -79,6 +79,11 @@ (unless (hashtable-ref var-table k #f) (hashtable-set! var-table k (make-var-cell ns name jolt-unbound))))) +;; regex (jolt-i0s3): defines regex-t + the re-* fns (def-var!'d into +;; clojure.core), so it loads after def-var! and before the printer below (which +;; renders a regex-t as #"source"). +(load "host/chez/regex.ss") + ;; --- jolt number printing ---------------------------------------------------- ;; jolt models every number as a Clojure double: integer-valued values print ;; without a ".0" (the Janet host prints (* 1.0 5) as "5", (/ 1 2) as "0.5"). @@ -115,6 +120,7 @@ ((jolt-symbol? x) (let ((ns (symbol-t-ns x))) (if (or (jolt-nil? ns) (not ns) (eq? ns '())) (symbol-t-name x) (string-append ns "/" (symbol-t-name x))))) + ((regex-t? x) (string-append "#\"" (regex-t-source x) "\"")) ((pvec? x) (let ((acc '())) (let loop ((i (fx- (pvec-count x) 1))) (when (fx>=? i 0) (set! acc (cons (jolt-pr-str (pvec-nth-d x i jolt-nil)) acc)) (loop (fx- i 1)))) (string-append "[" (jolt-str-join acc) "]"))) diff --git a/jolt-core/jolt/analyzer.clj b/jolt-core/jolt/analyzer.clj index 7e1660d..6efd853 100644 --- a/jolt-core/jolt/analyzer.clj +++ b/jolt-core/jolt/analyzer.clj @@ -21,6 +21,7 @@ form-vec? form-map? form-set? form-char? form-literal? form-elements form-vec-items form-map-pairs form-set-items form-special? compile-ns + form-regex? form-regex-source form-macro? form-expand-1 resolve-global form-sym-meta host-intern! form-syntax-quote-lower record-type? record-ctor-key form-position]])) @@ -355,4 +356,8 @@ (form-map-pairs form))) (form-set? form) (set-node (mapv #(analyze ctx % env) (form-set-items form))) (form-list? form) (analyze-list ctx form env) + ;; regex literal #"…" -> a :regex IR node (leaf). The Janet back end punts it + ;; (interpreter compiles via the seed PEG engine); the Chez back end emits a + ;; jolt-regex value over the vendored irregex. + (form-regex? form) {:op :regex :source (form-regex-source form)} :else (uncompilable "unsupported form")))) diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index e699845..c3ef2a7 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -781,6 +781,9 @@ # punt to the interpreter, exactly as the analyzer used to before producing # a :host-call node (the Chez back end lowers it instead). :host-call (error "jolt/uncompilable: host method call") + # regex literal: the back end doesn't compile patterns — punt to the + # interpreter (the seed compiles #"…" to a Janet PEG). Chez emits jolt-regex. + :regex (error "jolt/uncompilable: regex literal") (error (string "backend: unhandled op " (node :op)))))) (defn emit-ir diff --git a/src/jolt/host_iface.janet b/src/jolt/host_iface.janet index 767821a..8edd912 100644 --- a/src/jolt/host_iface.janet +++ b/src/jolt/host_iface.janet @@ -46,6 +46,13 @@ (phm/phm? form))) (defn h-set? [form] (and (struct? form) (= :jolt/set (form :jolt/type)))) (defn h-char? [form] (and (struct? form) (= :jolt/char (form :jolt/type)))) +# A regex literal #"…" reads as a tagged form {:jolt/type :jolt/tagged :tag :regex +# :form "source"}. The analyzer lowers it to a :regex IR node (Chez emits a +# jolt-regex value; the Janet back end punts to the interpreter, which compiles it +# via the seed PEG engine). +(defn h-regex? [form] + (and (struct? form) (= :jolt/tagged (form :jolt/type)) (= :regex (form :tag)))) +(defn h-regex-source [form] (form :form)) (defn h-literal? [form] (or (nil? form) (boolean? form) (number? form) (string? form) @@ -254,6 +261,7 @@ "form-sym-meta" h-sym-meta "form-list?" h-list? "form-vec?" h-vector? "form-map?" h-map? "form-set?" h-set? "form-char?" h-char? "form-literal?" h-literal? + "form-regex?" h-regex? "form-regex-source" h-regex-source "form-elements" h-elements "form-vec-items" h-vector-items "form-map-pairs" h-map-pairs "form-set-items" h-set-items "form-special?" h-special? "compile-ns" h-current-ns "form-macro?" h-macro? diff --git a/test/chez/README.md b/test/chez/README.md index 8c5fb5e..4c94c83 100644 --- a/test/chez/README.md +++ b/test/chez/README.md @@ -72,19 +72,21 @@ catalogs the gaps; macros are skipped (analyze-time only, not a runtime value): JOLT_CHEZ_PRELUDE=1 janet test/chez/core-prelude-probe.janet -Baseline after inc 3h (host-interop method calls): **354/355 non-macro core forms -emit** to Scheme (was 348 at inc 3g, 342 at inc 3f). A `.method` call now analyzes -to a `:host-call` IR node; the Chez emitter lowers it to a `jolt-host-call` -dispatch for the methods the RT shims — `.write` → port `display`, `.isDirectory` -→ `file-directory?`, `.listFiles` → `directory-list` — closing the io tier's -print-method defmethods and `file-seq` (now 20/20). Any other method is out of -subset (a clean emit-time reject, so it can't masquerade as a compiled-but-broken -divergence); the Janet back end punts ALL `:host-call` to the interpreter. Prior -incs: `:quote` reconstructs the raw reader form as RT constructors; `:throw` → -`jolt-throw`, `:try` → `guard` + `dynamic-wind`, `ex-info` native-op; `letfn` → -`letrec*`; `declare`/def-no-init → a reserved var cell. Remaining 1 gap: the regex -literal in `parse-uuid` (needs a regex engine on Chez — see jolt issue). The probe -has a regression floor (354). +Baseline after inc 3i (regex): **355/355 non-macro core forms emit** to Scheme — +the whole non-macro clojure.core now lowers. inc 3i closed the last gap, the regex +literal in `parse-uuid`: a `#"…"` literal lowers to a `:regex` IR node and the Chez +emitter emits a `jolt-regex` value over **vendored irregex** (Alex Shinn, BSD, +`vendor/irregex` submodule) — a portable Scheme regex with PCRE/Java-style string +patterns. `re-pattern`/`re-matches`/`re-find`/`re-seq`/`regex?` are `def-var!`'d +into clojure.core (`host/chez/regex.ss`); they stay OUT of the subset native-ops +(irregex's Unicode/property-class semantics differ from the seed's byte-PEG +approximation), so they resolve in prelude mode — the path the assembled prelude +takes — without dragging engine-difference divergences into the subset corpus. The +Janet back end punts `:regex` to the interpreter (the seed compiles `#"…"` to a +Janet PEG). Prior incs: inc 3h `.method` → `:host-call` (`jolt-host-call` for +`.write`/`.isDirectory`/`.listFiles`); `:quote`, `:throw`, `:try`, `ex-info`, +`letfn` → `letrec*`, `declare`/def-no-init → reserved var cell. The probe has a +regression floor (355) — every non-macro core form must keep emitting. Prior, inc 3b (seq tier + dynamic IFn, jolt-5pso): 595/595 compiled, 0 divergences, 2060/2655 out of subset. The seq tier brought up a list/lazy-seq type with diff --git a/test/chez/core-prelude-probe.janet b/test/chez/core-prelude-probe.janet index ae3f986..f1ad0c5 100644 --- a/test/chez/core-prelude-probe.janet +++ b/test/chez/core-prelude-probe.janet @@ -98,7 +98,7 @@ # Regression floor (raise it as new IR ops / RT shims land, like the suite # baseline). Fails if prelude emit reach drops below the recorded baseline. -(def reach-floor 354) +(def reach-floor 355) (when (< compiled reach-floor) (printf "REGRESSION: prelude emit reach %d < floor %d" compiled reach-floor) (os/exit 1)) diff --git a/test/chez/emit-test.janet b/test/chez/emit-test.janet index 4d87830..07ba031 100644 --- a/test/chez/emit-test.janet +++ b/test/chez/emit-test.janet @@ -300,6 +300,42 @@ (ok "runtime .isDirectory \"/\" = true" (and (= code 0) (= out "true")) (string "chez=" out " | " err))) +# 3m) regex (jolt-i0s3): the #"…" literal lowers to a jolt-regex value over the +# vendored irregex; re-pattern/re-matches/re-find/re-seq/regex? are def-var!'d +# into clojure.core (not subset native-ops — irregex's Unicode/property +# semantics differ from the seed's byte-PEG), so they resolve in PRELUDE mode, +# the path the assembled prelude takes. Parity vs the CLI oracle on standard +# PCRE patterns both engines agree on. +(defn run-prelude [src] + (emit/set-prelude-mode! true) + (def r (protect (emit/emit (backend/analyze-form ctx (in (r/parse-next src) 0))))) + (emit/set-prelude-mode! false) + (if (not (r 0)) [:emit-err (r 1) ""] + (do + (spit "/tmp/chez-regex-prelude.ss" (emit/program @[] (r 1))) + (def proc (os/spawn ["chez" "--script" "/tmp/chez-regex-prelude.ss"] :p {:out :pipe :err :pipe})) + (def out (ev/read (proc :out) 0x100000)) + (def err (ev/read (proc :err) 0x100000)) + [(os/proc-wait proc) (string/trim (if out (string out) "")) (string/trim (if err (string err) ""))]))) + +# bare #"…" literal runs in plain subset mode (the :regex node needs no core fn). +(each src ["#\"\\d+\"" "(do #\"a.c\")"] + (let [[code out err] (d/run-on-chez ctx src) want (cli-oracle src)] + (ok (string "regex literal: " src) (and (= code 0) (= out want)) + (string "chez=" out " janet=" want " | " err)))) + +# re-* surface via prelude mode (def-var!'d fns), parity vs the CLI oracle. +(each src ["(re-matches #\"\\d+\" \"123\")" + "(re-matches #\"\\d+\" \"12a\")" + "(re-find #\"\\d+\" \"abc123def\")" + "(re-find #\"([a-z])(\\d)\" \"--a1--\")" + "(re-seq #\"\\d+\" \"a1b22c333\")" + "(regex? #\"\\d+\")" + "(re-matches #\"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\" \"550e8400-e29b-41d4-a716-446655440000\")"] + (let [[code out err] (run-prelude src) want (cli-oracle src)] + (ok (string "regex: " src) (and (= code 0) (= out want)) + (string "chez=" out " janet=" want " | " err)))) + # 3h) prelude mode (inc 3d): emitting clojure.core ITSELF, a core->core ref must # lower to a runtime var-deref instead of being rejected as "out of subset". # `frequencies` is a core fn but not a native-op, so it exercises the switch. diff --git a/vendor/irregex b/vendor/irregex new file mode 160000 index 0000000..c948a70 --- /dev/null +++ b/vendor/irregex @@ -0,0 +1 @@ +Subproject commit c948a704fc732914a243c1643bfe359913d11c7b