From 39cb78a12d8e35be69a3ddcef210afc5e5843552 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Tue, 16 Jun 2026 23:49:15 -0400 Subject: [PATCH] regex: single-digit backreferences \1..\9 (jolt-xtss) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit \1..\9 now match the text captured by the corresponding group, so patterns like ([-*_])\1\1 or (\w+) \1 work. The parser records which groups are referenced; a referenced group additionally captures its text under a tag and the backref compiles to a PEG (backmatch). Only referenced groups change — they match possessively (the CPS-over-possessive-PEG engine can't backtrack into a tagged capture), so backtracking back into a backreferenced group isn't supported (rare). Unreferenced groups keep full backtracking and position-based result capture unchanged. --- src/jolt/regex.janet | 44 +++++++++++++++++++++++++++----------- test/spec/regex-spec.janet | 9 ++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/jolt/regex.janet b/src/jolt/regex.janet index 2e3ccce..d0e12cf 100644 --- a/src/jolt/regex.janet +++ b/src/jolt/regex.janet @@ -8,8 +8,10 @@ # Supported: literals, `.`, char classes `[...]`/`[^...]` (ranges, POSIX, # `\d \w \s` etc.), quantifiers `* + ? {n} {n,} {n,m}` (greedy + lazy `?`), # groups `(...)` (numbered), non-capturing `(?:...)`, lookahead `(?=...)`/`(?!...)`, -# alternation `|`, anchors `^ $ \b \B`, escapes, inline flag `(?i)`. -# Not supported: lookbehind, backreferences, named groups (rare; documented). +# alternation `|`, anchors `^ $ \b \B`, escapes, inline flag `(?i)`, and +# single-digit backreferences `\1`..`\9` (a backreferenced group matches +# possessively — no backtracking back into it; rare in practice). +# Not supported: lookbehind, named groups (rare; documented). (defn- chr [s] (get s 0)) (defn regex? [x] (and (table? x) (= :jolt/regex (x :jolt/type)))) @@ -189,6 +191,10 @@ p {:op :pred :peg p} (= nc (chr "b")) {:op :anchor :kind :wordb} (= nc (chr "B")) {:op :anchor :kind :nwordb} + # \1..\9 backreference (single digit) — record the referenced group so + # the emitter knows to tag-capture its text + (and (>= nc (chr "1")) (<= nc (chr "9"))) + (let [n (- nc (chr "0"))] (put (st :refs) n true) {:op :backref :n n}) {:op :char :b (esc-byte nc) :ci ci}))) (= c (chr "^")) (do (++ (st :pos)) {:op :anchor :kind :start}) (= c (chr "$")) (do (++ (st :pos)) {:op :anchor :kind :end}) @@ -241,9 +247,9 @@ (if (= 1 (length branches)) (branches 0) {:op :alt :items branches}))) (defn- parse [source] - (def st @{:s source :pos 0 :ngroup 0 :ci false :ml false :dotall false}) + (def st @{:s source :pos 0 :ngroup 0 :ci false :ml false :dotall false :refs @{}}) (def ast (parse-alt st)) - [ast (st :ngroup) (st :ml)]) + [ast (st :ngroup) (st :ml) (st :refs)]) # ============================================================ # Emit: AST -> PEG grammar (continuation passing) @@ -256,7 +262,7 @@ ~(set ,(string/from-bytes (lower-b b) (upper-b b))) ~(set ,(string/from-bytes b)))) -(defn- make-emitter [grammar ml] +(defn- make-emitter [grammar ml refs] (var ctr 0) (defn fresh [] (++ ctr) (keyword (string "r" ctr))) (var emit nil) @@ -310,9 +316,20 @@ (while (> c 0) (set acc (emit item acc)) (-- c)) acc) :group (let [n (ast :n)] - ~(sequence (/ (position) ,(fn [p] [n :s p])) - ,(emit (ast :item) - ~(sequence (/ (position) ,(fn [p] [n :e p])) ,k)))) + (if (and refs (refs n)) + # backreferenced group: also capture its text under tag :gN so a + # later \N can (backmatch). The item runs with just the end-marker + # as its continuation, so it matches POSSESSIVELY — backtracking + # INTO a backreferenced group isn't supported (rare; the usual + # (x)...\1 patterns don't need it). + (let [tag (keyword (string "g" n))] + ~(sequence (/ (position) ,(fn [p] [n :s p])) + (<- ,(emit (ast :item) ~(/ (position) ,(fn [p] [n :e p]))) ,tag) + ,k)) + ~(sequence (/ (position) ,(fn [p] [n :s p])) + ,(emit (ast :item) + ~(sequence (/ (position) ,(fn [p] [n :e p])) ,k))))) + :backref ~(sequence (backmatch ,(keyword (string "g" (ast :n)))) ,k) :ncgroup (emit (ast :item) k) :look (if (ast :neg) ~(sequence (not ,(emit (ast :item) 0)) ,k) @@ -335,9 +352,9 @@ emit) (defn compile-regex [source] - (def [ast ngroups ml] (parse source)) + (def [ast ngroups ml refs] (parse source)) (def grammar @{}) - (def emit (make-emitter grammar ml)) + (def emit (make-emitter grammar ml refs)) # group 0 = whole match: mark start, body, mark end (def body (emit ast ~(sequence (/ (position) ,(fn [p] [0 :e p])) 0))) (put grammar :main ~(sequence (/ (position) ,(fn [p] [0 :s p])) ,body)) @@ -362,8 +379,11 @@ (def starts (array/new-filled (+ ngroups 1))) (def ends (array/new-filled (+ ngroups 1))) (each m marks - (let [n (m 0)] - (if (= (m 1) :s) (put starts n (m 2)) (put ends n (m 2))))) + # backreferenced groups also leave a tagged TEXT capture (a string) on the + # stack — skip those; only the [n :s/:e pos] position tuples carry group spans + (when (indexed? m) + (let [n (m 0)] + (if (= (m 1) :s) (put starts n (m 2)) (put ends n (m 2)))))) (def groups (array/new-filled (+ ngroups 1))) (var n 0) (while (<= n ngroups) diff --git a/test/spec/regex-spec.janet b/test/spec/regex-spec.janet index 777a5e6..78de377 100644 --- a/test/spec/regex-spec.janet +++ b/test/spec/regex-spec.janet @@ -72,3 +72,12 @@ # expand to ~2^61 nodes and hang at compile time. ["nested bounds compile" "true" "(boolean (re-matches #\"[a-z](?:[a-z]{0,61}[a-z])?(?:-[a-z]{0,61}[a-z])*\" \"abc-def\"))"]) + +(defspec "regex / backreferences (jolt-xtss)" + ["repeated char" "true" "(boolean (re-find #\"(.)\\1\" \"abba\"))"] + ["no repeat = nil" "nil" "(re-find #\"(.)\\1\" \"abc\")"] + ["thematic-break run" "true" "(boolean (re-matches #\"([-*_])\\1\\1\" \"---\"))"] + ["mismatched run" "nil" "(re-matches #\"([-*_])\\1\\1\" \"-*_\")"] + ["repeated word" "[\"the the\" \"the\"]" "(re-find #\"(\\w+) \\1\" \"say the the word\")"] + ["group still captures" "[\"x=x\" \"x\"]" "(re-matches #\"(\\w+)=\\1\" \"x=x\")"] + ["html tag pair" "true" "(boolean (re-matches #\"<(\\w+)>.*\" \"hi\"))"])