core: seq walks over concrete collections are linear (bench TOTAL -18%)
Reviewing flatiron's morsel batching for applicability turned up something better hiding under the lazy machinery: every cell over a concrete collection was produced by slicing the REMAINDER ((tuple/slice c 1) per element in coll->cells, and rest/next sliced the same way), so any full walk was O(n^2). mapv over 40k elements took 10.4 SECONDS; a 20k-element first/rest loop took two. Cells over indexed collections now walk by INDEX (one shared indexed-cells helper, O(1) per step), and rest/next of a vector/tuple/list return an O(1) lazy view from index 1 — which also makes (rest [1 2 3]) a SEQ, as in Clojure (it was a vector-typed slice; seq?/vector? rows pin the change). mapv 40k: 10405 -> 182 ms. rest-loop 20k: 2040 -> 31 ms. Whole bench: seq-pipe -28%, into-vec -24%, str-join -18%, hof -26%, TOTAL 4565 -> 3753 (-18% vs main, back to back). Chunked seqs (jolt-yqc) drop in priority: the quadratic walks were the actual cost; chunking now only amortizes per-element closure allocation. Nine regression rows incl. 20k/50k linear-scaling smoke tests. Gate exit 0.
This commit is contained in:
parent
c24c7617b8
commit
4f78f6be33
2 changed files with 38 additions and 6 deletions
|
|
@ -56,6 +56,13 @@
|
||||||
(defn sorted-entries-arr [coll]
|
(defn sorted-entries-arr [coll]
|
||||||
(let [e (coll :entries)] (if (pvec? e) (pv->array e) e)))
|
(let [e (coll :entries)] (if (pvec? e) (pv->array e) e)))
|
||||||
|
|
||||||
|
# Lazy cell chain over an indexed (tuple/array) collection, walking by INDEX —
|
||||||
|
# O(1) per step. Slicing the remainder per step (the old shape) made every
|
||||||
|
# full walk over a concrete collection O(n^2).
|
||||||
|
(defn indexed-cells [t i]
|
||||||
|
(if (>= i (length t)) nil
|
||||||
|
@[(in t i) (fn [] (indexed-cells t (+ i 1)))]))
|
||||||
|
|
||||||
# Canonicalize a collection key/element to a value-hashable Janet struct/tuple so
|
# Canonicalize a collection key/element to a value-hashable Janet struct/tuple so
|
||||||
# the PHM/PHS treat value-equal maps/vectors as the same key (Janet hashes tables
|
# the PHM/PHS treat value-equal maps/vectors as the same key (Janet hashes tables
|
||||||
# by identity otherwise). Installed into phm via set-canonicalize-key!.
|
# by identity otherwise). Installed into phm via set-canonicalize-key!.
|
||||||
|
|
@ -699,11 +706,18 @@
|
||||||
# rest never returns nil — Clojure's rest yields () on an exhausted seq.
|
# rest never returns nil — Clojure's rest yields () on an exhausted seq.
|
||||||
(lazy-seq? coll) (let [r (ls-rest coll)] (if (nil? r) @[] r))
|
(lazy-seq? coll) (let [r (ls-rest coll)] (if (nil? r) @[] r))
|
||||||
(plist? coll) (pl-rest coll)
|
(plist? coll) (pl-rest coll)
|
||||||
(pvec? coll) (let [a (pv->array coll)] (if (<= (length a) 1) @[] (array/slice a 1)))
|
# Indexed collections: an O(1) lazy view from index 1 (Clojure: rest of a
|
||||||
|
# vector is a seq, not a vector). Slicing per step made first/rest loops
|
||||||
|
# over concrete collections O(n^2) — a 20k rest-loop took two seconds.
|
||||||
|
(pvec? coll) (let [a (pv->array coll)]
|
||||||
|
(if (<= (length a) 1) @[]
|
||||||
|
(make-lazy-seq (fn [] (indexed-cells a 1)))))
|
||||||
(or (nil? coll) (= 0 (length coll))) @[]
|
(or (nil? coll) (= 0 (length coll))) @[]
|
||||||
(string? coll) (tuple ;(map make-char (string/bytes (string/slice coll 1))))
|
(string? coll) (tuple ;(map make-char (string/bytes (string/slice coll 1))))
|
||||||
(tuple? coll) (tuple/slice coll 1)
|
(tuple? coll) (if (<= (length coll) 1) @[]
|
||||||
(array/slice coll 1)))
|
(make-lazy-seq (fn [] (indexed-cells coll 1))))
|
||||||
|
(if (<= (length coll) 1) @[]
|
||||||
|
(make-lazy-seq (fn [] (indexed-cells coll 1))))))
|
||||||
|
|
||||||
(defn core-next [coll]
|
(defn core-next [coll]
|
||||||
# next is rest, but nil when the rest is empty. seq-done? realizes one lazy
|
# next is rest, but nil when the rest is empty. seq-done? realizes one lazy
|
||||||
|
|
@ -952,14 +966,15 @@
|
||||||
(if (= :jolt/pending cell) nil cell))
|
(if (= :jolt/pending cell) nil cell))
|
||||||
(if (tuple? c)
|
(if (tuple? c)
|
||||||
# user sequential data: every element is a value, no cell-detection.
|
# user sequential data: every element is a value, no cell-detection.
|
||||||
(if (= 0 (length c)) nil
|
# indexed-cells walks by INDEX — the old (tuple/slice c 1) per cell
|
||||||
@[(in c 0) (fn [] (coll->cells (tuple/slice c 1)))])
|
# made any walk over a concrete collection O(n^2).
|
||||||
|
(if (= 0 (length c)) nil (indexed-cells c 0))
|
||||||
(if (array? c)
|
(if (array? c)
|
||||||
# mutable array: a genuine cons cell, or an eager seq result.
|
# mutable array: a genuine cons cell, or an eager seq result.
|
||||||
(if (= 0 (length c)) nil
|
(if (= 0 (length c)) nil
|
||||||
(if (and (= 2 (length c)) (function? (in c 1)))
|
(if (and (= 2 (length c)) (function? (in c 1)))
|
||||||
c # already a cell [val, rest-thunk]
|
c # already a cell [val, rest-thunk]
|
||||||
@[(in c 0) (fn [] (coll->cells (array/slice c 1)))]))
|
(indexed-cells c 0)))
|
||||||
# Other concrete seqables (set/map/sorted coll/string/buffer): coerce
|
# Other concrete seqables (set/map/sorted coll/string/buffer): coerce
|
||||||
# to a tuple seq via core-seq, then recurse. (lazy/indexed above.)
|
# to a tuple seq via core-seq, then recurse. (lazy/indexed above.)
|
||||||
(if (or (set? c) (phm? c) (buffer? c) (string? c) (core-sorted? c)
|
(if (or (set? c) (phm? c) (buffer? c) (string? c) (core-sorted? c)
|
||||||
|
|
|
||||||
|
|
@ -273,3 +273,20 @@
|
||||||
["splitv-at" "[[1 2] [3 4]]" "(splitv-at 2 [1 2 3 4])"]
|
["splitv-at" "[[1 2] [3 4]]" "(splitv-at 2 [1 2 3 4])"]
|
||||||
["splitv-at first is vector" "true" "(vector? (first (splitv-at 2 [1 2 3])))"]
|
["splitv-at first is vector" "true" "(vector? (first (splitv-at 2 [1 2 3])))"]
|
||||||
["splitv-at past end" "[[1 2] []]" "(splitv-at 5 [1 2])"])
|
["splitv-at past end" "[[1 2] []]" "(splitv-at 5 [1 2])"])
|
||||||
|
|
||||||
|
# Linear seq walks (flatiron review find): cell chains over concrete
|
||||||
|
# collections walk by INDEX. Slicing the remainder per step made every full
|
||||||
|
# walk O(n^2) — mapv over 40k elements took ten seconds, a 20k first/rest
|
||||||
|
# loop took two. rest/next of an indexed collection is an O(1) lazy view —
|
||||||
|
# and therefore a SEQ, as in Clojure (it was a vector-typed slice).
|
||||||
|
(defspec "sequences / linear walks over concrete collections"
|
||||||
|
["rest of vector is a seq" "true" "(seq? (rest [1 2 3]))"]
|
||||||
|
["rest of vector not vector" "false" "(vector? (rest [1 2 3]))"]
|
||||||
|
["rest values" "(quote (2 3))" "(rest [1 2 3])"]
|
||||||
|
["next of vector" "(quote (2 3))" "(next [1 2 3])"]
|
||||||
|
["next exhausts to nil" "nil" "(next [1])"]
|
||||||
|
["rest exhausts to ()" "()" "(rest [1])"]
|
||||||
|
["rest-loop scales (20k linear)" "20000"
|
||||||
|
"(loop [xs (seq (vec (range 20000))) n 0] (if xs (recur (next xs) (inc n)) n))"]
|
||||||
|
["mapv scales (50k linear)" "50000" "(count (mapv inc (vec (range 50000))))"]
|
||||||
|
["nested walk" "[2 3]" "(vec (rest (mapv inc [0 1 2])))"])
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue