From 4f78f6be33f62e67c7bc41cd8e38fff2bc1d0c1a Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 10 Jun 2026 18:56:16 -0400 Subject: [PATCH] core: seq walks over concrete collections are linear (bench TOTAL -18%) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewing flatiron's morsel batching for applicability turned up something better hiding under the lazy machinery: every cell over a concrete collection was produced by slicing the REMAINDER ((tuple/slice c 1) per element in coll->cells, and rest/next sliced the same way), so any full walk was O(n^2). mapv over 40k elements took 10.4 SECONDS; a 20k-element first/rest loop took two. Cells over indexed collections now walk by INDEX (one shared indexed-cells helper, O(1) per step), and rest/next of a vector/tuple/list return an O(1) lazy view from index 1 — which also makes (rest [1 2 3]) a SEQ, as in Clojure (it was a vector-typed slice; seq?/vector? rows pin the change). mapv 40k: 10405 -> 182 ms. rest-loop 20k: 2040 -> 31 ms. Whole bench: seq-pipe -28%, into-vec -24%, str-join -18%, hof -26%, TOTAL 4565 -> 3753 (-18% vs main, back to back). Chunked seqs (jolt-yqc) drop in priority: the quadratic walks were the actual cost; chunking now only amortizes per-element closure allocation. Nine regression rows incl. 20k/50k linear-scaling smoke tests. Gate exit 0. --- src/jolt/core.janet | 27 +++++++++++++++++++++------ test/spec/sequences-spec.janet | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/jolt/core.janet b/src/jolt/core.janet index cbd135b..b5a39ac 100644 --- a/src/jolt/core.janet +++ b/src/jolt/core.janet @@ -56,6 +56,13 @@ (defn sorted-entries-arr [coll] (let [e (coll :entries)] (if (pvec? e) (pv->array e) e))) +# Lazy cell chain over an indexed (tuple/array) collection, walking by INDEX — +# O(1) per step. Slicing the remainder per step (the old shape) made every +# full walk over a concrete collection O(n^2). +(defn indexed-cells [t i] + (if (>= i (length t)) nil + @[(in t i) (fn [] (indexed-cells t (+ i 1)))])) + # Canonicalize a collection key/element to a value-hashable Janet struct/tuple so # the PHM/PHS treat value-equal maps/vectors as the same key (Janet hashes tables # by identity otherwise). Installed into phm via set-canonicalize-key!. @@ -699,11 +706,18 @@ # rest never returns nil — Clojure's rest yields () on an exhausted seq. (lazy-seq? coll) (let [r (ls-rest coll)] (if (nil? r) @[] r)) (plist? coll) (pl-rest coll) - (pvec? coll) (let [a (pv->array coll)] (if (<= (length a) 1) @[] (array/slice a 1))) + # Indexed collections: an O(1) lazy view from index 1 (Clojure: rest of a + # vector is a seq, not a vector). Slicing per step made first/rest loops + # over concrete collections O(n^2) — a 20k rest-loop took two seconds. + (pvec? coll) (let [a (pv->array coll)] + (if (<= (length a) 1) @[] + (make-lazy-seq (fn [] (indexed-cells a 1))))) (or (nil? coll) (= 0 (length coll))) @[] (string? coll) (tuple ;(map make-char (string/bytes (string/slice coll 1)))) - (tuple? coll) (tuple/slice coll 1) - (array/slice coll 1))) + (tuple? coll) (if (<= (length coll) 1) @[] + (make-lazy-seq (fn [] (indexed-cells coll 1)))) + (if (<= (length coll) 1) @[] + (make-lazy-seq (fn [] (indexed-cells coll 1)))))) (defn core-next [coll] # next is rest, but nil when the rest is empty. seq-done? realizes one lazy @@ -952,14 +966,15 @@ (if (= :jolt/pending cell) nil cell)) (if (tuple? c) # user sequential data: every element is a value, no cell-detection. - (if (= 0 (length c)) nil - @[(in c 0) (fn [] (coll->cells (tuple/slice c 1)))]) + # indexed-cells walks by INDEX — the old (tuple/slice c 1) per cell + # made any walk over a concrete collection O(n^2). + (if (= 0 (length c)) nil (indexed-cells c 0)) (if (array? c) # mutable array: a genuine cons cell, or an eager seq result. (if (= 0 (length c)) nil (if (and (= 2 (length c)) (function? (in c 1))) c # already a cell [val, rest-thunk] - @[(in c 0) (fn [] (coll->cells (array/slice c 1)))])) + (indexed-cells c 0))) # Other concrete seqables (set/map/sorted coll/string/buffer): coerce # to a tuple seq via core-seq, then recurse. (lazy/indexed above.) (if (or (set? c) (phm? c) (buffer? c) (string? c) (core-sorted? c) diff --git a/test/spec/sequences-spec.janet b/test/spec/sequences-spec.janet index 4d136fa..f851a75 100644 --- a/test/spec/sequences-spec.janet +++ b/test/spec/sequences-spec.janet @@ -273,3 +273,20 @@ ["splitv-at" "[[1 2] [3 4]]" "(splitv-at 2 [1 2 3 4])"] ["splitv-at first is vector" "true" "(vector? (first (splitv-at 2 [1 2 3])))"] ["splitv-at past end" "[[1 2] []]" "(splitv-at 5 [1 2])"]) + +# Linear seq walks (flatiron review find): cell chains over concrete +# collections walk by INDEX. Slicing the remainder per step made every full +# walk O(n^2) — mapv over 40k elements took ten seconds, a 20k first/rest +# loop took two. rest/next of an indexed collection is an O(1) lazy view — +# and therefore a SEQ, as in Clojure (it was a vector-typed slice). +(defspec "sequences / linear walks over concrete collections" + ["rest of vector is a seq" "true" "(seq? (rest [1 2 3]))"] + ["rest of vector not vector" "false" "(vector? (rest [1 2 3]))"] + ["rest values" "(quote (2 3))" "(rest [1 2 3])"] + ["next of vector" "(quote (2 3))" "(next [1 2 3])"] + ["next exhausts to nil" "nil" "(next [1])"] + ["rest exhausts to ()" "()" "(rest [1])"] + ["rest-loop scales (20k linear)" "20000" + "(loop [xs (seq (vec (range 20000))) n 0] (if xs (recur (next xs) (inc n)) n))"] + ["mapv scales (50k linear)" "50000" "(count (mapv inc (vec (range 50000))))"] + ["nested walk" "[2 3]" "(vec (rest (mapv inc [0 1 2])))"])