From e377c223b63e4cc865a48e0970a2dfc122b87249 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Sat, 13 Jun 2026 17:01:35 -0400 Subject: [PATCH] wip: generalize shape mechanism off the hardcoded vec3 shape (jolt-t34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the {:r :g :b} hardcoding. ANY constant key set is now a shape: - inference: a struct type from a map LITERAL carries :shape (its canonical str-sorted key vector — completeness); joins/access-inferred structs lack it, so they never get a bare index. The literal node and lookup subjects carry the shape; the back end derives the index from it. - backend: emit-map turns any shape-tagged const-key map into a shape tuple; emit-kw-lookup reads the field by bare index when the complete shape is proven, else by the value's own descriptor (so a shape-rec whose :shape was dropped by a join still reads correctly). - runtime: core-get and core-assoc handle shape-recs. Status: CORRECT for direct field access, container round-trips, and assoc (minimal repros pass). NOT yet complete — the full ray tracer still hits an uncovered path (a shape-rec reaching a map op without coverage: keys/vals/ count/seq/equality/print/jolt-call/dissoc/contains?/the interpreter's coll-lookup all still need shape-rec branches). And the perf win needs COMPLETENESS PRESERVATION through joins/containers (merge-fields/cap drop :shape today, so nested vec3 access falls to the descriptor path, slower than a struct get) — without it the general version is slower than the vec3 prototype. All behind JOLT_SHAPE (off by default). Gate green with the flag off, suite 4718. This preserves the general design; the transparency layer + completeness preservation are the remaining multi-session work. --- jolt-core/jolt/passes.clj | 46 ++++++++++++++---------- src/jolt/backend.janet | 46 +++++++++++++----------- src/jolt/core.janet | 76 ++++++++++++++++++++------------------- 3 files changed, 93 insertions(+), 75 deletions(-) diff --git a/jolt-core/jolt/passes.clj b/jolt-core/jolt/passes.clj index 6214e49..9646e10 100644 --- a/jolt-core/jolt/passes.clj +++ b/jolt-core/jolt/passes.clj @@ -806,21 +806,26 @@ ;; k, if known, else :any. (defn- struct-safe? [t] (struct-type? t)) (defn- field-type [t k] (if (struct-type? t) (get (sfields t) k :any) :any)) -;; vec3 shape detection (jolt-t34, prototype): a struct type whose key set is -;; exactly {:r :g :b}. The back end represents such maps as shape tuples, so a -;; lookup on a value PROVEN to be this shape can read by bare index with no -;; runtime check. Scoped to the one shape for the prototype. -(defn- vec3-shape? [t] - (and (struct-type? t) - (let [fs (sfields t)] - (and (get fs :r) (get fs :g) (get fs :b) (= 3 (count (keys fs))))))) -;; the structural hint for a subject: :shape when provably the vec3 shape (bare -;; indexed read), else :struct when raw-get-safe. -(defn- struct-hint [t] (if (vec3-shape? t) :shape :struct)) +;; Shape (hidden class, jolt-t34). A struct type built from a map LITERAL carries +;; its complete layout — :shape, the canonical (str-sorted) key vector. The back +;; end represents such a map as a shape tuple and reads a field by bare index. +;; A struct type from a JOIN or from field-access inference has no :shape +;; (incomplete: the full key set isn't proven), so it keeps the dynamic path — +;; never a bare index. No shape is hardcoded; any constant key set is one. +(defn- shape-order + "Canonical key order for a shape: keys sorted by their string form, so two + literals with the same keys in any order intern to the same shape." + [ks] (vec (sort (fn [a b] (compare (str a) (str b))) ks))) +(defn- type-shape [t] (get t :shape)) ;; tag a node (any expression, not just a :local) so the back end can specialize ;; a lookup whose SUBJECT is that node — this is what makes nested access work: ;; (:direction ray) is tagged struct, so (:r (:direction ray)) drops its guard. (defn- mark-hint [node h] (assoc node :hint h)) +;; tag a lookup subject as a struct, carrying the complete shape when known +;; (so the back end bare-indexes) — jolt-t34 +(defn- mark-struct [node t] + (let [n (assoc node :hint :struct)] + (if (get t :shape) (assoc n :shape (get t :shape)) n))) ;; a value provably neither nil nor false — the back end only builds a struct ;; (vs a phm) when every value is non-nil/non-false, so a map literal is a struct ;; only when all its values have such a type. Collections are non-nil. @@ -940,7 +945,8 @@ (let [t (get tenv (get node :name))] [(if t t :any) (cond - (struct-safe? t) (assoc node :hint (struct-hint t)) + (struct-safe? t) (let [n (assoc node :hint :struct)] + (if (type-shape t) (assoc n :shape (type-shape t)) n)) (vec-type? t) (assoc node :hint :vector) :else node)]) (= op :map) @@ -953,10 +959,14 @@ struct? (and (> (count res) 0) (every? (fn [pr] (scalar-const? (nth pr 0))) pairs) (every? (fn [r] (truthy-type? (nth r 2))) res)) - t (if struct? - (cap (mk-struct (reduce (fn [m r] (assoc m (nth r 3) (nth r 2))) {} res)) type-depth) - :any)] - [t (assoc node :pairs (mapv (fn [r] [(nth r 0) (nth r 1)]) res))]) + base (when struct? + (cap (mk-struct (reduce (fn [m r] (assoc m (nth r 3) (nth r 2))) {} res)) type-depth)) + ;; a literal is a COMPLETE shape: carry its sorted key vector so the + ;; back end can lay it out and bare-index lookups (jolt-t34) + shp (when (and base (struct-type? base)) (shape-order (keys (sfields base)))) + t (if base (if shp (assoc base :shape shp) base) :any) + node' (assoc node :pairs (mapv (fn [r] [(nth r 0) (nth r 1)]) res))] + [t (if shp (assoc node' :shape shp) node')]) (= op :vector) (let [irs (mapv (fn [x] (infer x tenv)) (get node :items)) ets (mapv (fn [r] (nth r 0)) irs) @@ -998,7 +1008,7 @@ (and (= :const (get fnode :op)) (keyword? (get fnode :val)) (>= n 1) (<= n 2)) (let [mr (infer (nth args 0) tenv) mt (nth mr 0) - msub (if (struct-safe? mt) (mark-hint (nth mr 1) (struct-hint mt)) (nth mr 1)) + msub (if (struct-safe? mt) (mark-struct (nth mr 1) mt) (nth mr 1)) ft (field-type mt (get fnode :val)) dr (when (= n 2) (infer (nth args 1) tenv))] [(if dr (join ft (nth dr 0)) ft) @@ -1009,7 +1019,7 @@ (>= n 2) (= :const (get (nth args 1) :op)) (keyword? (get (nth args 1) :val))) (let [mr (infer (nth args 0) tenv) mt (nth mr 0) - msub (if (struct-safe? mt) (mark-hint (nth mr 1) (struct-hint mt)) (nth mr 1)) + msub (if (struct-safe? mt) (mark-struct (nth mr 1) mt) (nth mr 1)) kr (infer (nth args 1) tenv) ft (field-type mt (get (nth args 1) :val)) dr (when (= n 3) (infer (nth args 2) tenv))] diff --git a/src/jolt/backend.janet b/src/jolt/backend.janet index b9c32c7..016b1ca 100644 --- a/src/jolt/backend.janet +++ b/src/jolt/backend.janet @@ -355,15 +355,13 @@ # - ^:struct / ^Record hinted subject: skip the guard, bare get (~20 vs ~36ns). # - hinted + JOLT_CHECK_HINTS: keep the guard but THROW on the tagged arm, so a # lying hint surfaces a clear error (dev aid; off by default, no perf cost). -# vec3 shape layout: descriptor at 0, then sorted keys [:b :g :r] at 1,2,3 (jolt-t34) -(def- vec3-shape-idx {:b 1 :g 2 :r 3}) (defn- emit-kw-lookup [subj-node m-expr k d-expr] # the subject is a struct (raw-get-safe) when hinted so — by an explicit # ^:struct/^Record hint on a local, OR by inference tagging ANY subject # expression it proved to be a struct (jolt-d6u/RFC 0005), which is what lets # nested access like (:r (:direction ray)) drop its guard. - (def hinted (and subj-node (or (= :struct (subj-node :hint)) (= :shape (subj-node :hint))))) - (def checked (and (= :struct (subj-node :hint)) (os/getenv "JOLT_CHECK_HINTS"))) + (def hinted (and subj-node (= :struct (subj-node :hint)))) + (def checked (and hinted (os/getenv "JOLT_CHECK_HINTS"))) (def m (if (symbol? m-expr) m-expr (jsym))) (def wrap (fn [body] (if (symbol? m-expr) body ['let [m m-expr] body]))) (def err (when checked @@ -371,14 +369,22 @@ k " " (subj-node :name) ") — value carries :jolt/type " "(a phm/sorted/transient/lazy-seq), not the plain " "struct/record the ^:struct/^Record hint asserts")])) - (def sidx (and (os/getenv "JOLT_SHAPE") (get vec3-shape-idx k))) - # subject PROVEN to be the vec3 shape (jolt-t34): read by bare index, no check. - (def shaped (and sidx (= :shape (subj-node :hint)))) - # otherwise, when JOLT_SHAPE is on and the key is a vec3 field but the shape - # isn't proven, runtime-check (tuple? + struct? of elem 0, inline opcodes). + # Subject carries a complete :shape (jolt-t34) => it is provably a shape-rec; + # the field reads by bare index. The shape is the inference's canonical key + # vector, so the index is the key's position in it, +1 for the descriptor. + (def sidx + (when (and (os/getenv "JOLT_SHAPE") subj-node (subj-node :shape)) + (def sk (let [s (subj-node :shape)] (if (pv/pvec? s) (pv/pv->array s) s))) + (var pos nil) (var i 0) + (each kk sk (when (= kk k) (set pos i)) (++ i)) + (when pos (+ pos 1)))) + # sidx: complete shape proven -> bare index (fastest). Otherwise, with shapes + # on, a raw-get-safe value may still be a shape-rec (its :shape dropped by a + # join), so read the index from the value's own descriptor when it is a tuple; + # a real struct takes the bare get. (jolt-t34) (defn get-or-shape [getexpr] - (cond shaped ['in m sidx] - sidx ['if ['and ['tuple? m] ['struct? ['in m 0]]] ['in m sidx] getexpr] + (cond sidx ['in m sidx] + (os/getenv "JOLT_SHAPE") ['if ['tuple? m] ['in m ['+ 1 [[['in m 0] :idx] k]]] getexpr] getexpr)) (if (nil? d-expr) (let [fast (get-or-shape ['get m k])] @@ -481,17 +487,15 @@ (unless (and (= :const (k :op)) (or (keyword? kv) (string? kv) (number? kv) (boolean? kv))) (set fast false))) - # Shape-record fast path (jolt-t34, JOLT_SHAPE): a vec3-shaped {:r :g :b} map - # literal becomes a shape tuple (≈2x cheaper than a struct). Scoped to this - # one shape for the prototype so other maps (hit-info/ray/material) stay - # structs and assoc on them is unaffected. Values are emitted in the shape's - # canonical (sorted-key) order; element 0 is the compile-time shape descriptor. + # Shape-record fast path (jolt-t34, JOLT_SHAPE): a const-key map literal the + # inference tagged with a complete :shape becomes a shape tuple (≈2x cheaper + # than a struct). The shape is the inference's canonical (str-sorted) key + # vector — ANY constant key set, no hardcoding. Values are emitted in that + # order; element 0 is the compile-time shape descriptor. (def shape-keys - (when (and fast (os/getenv "JOLT_SHAPE")) - (def ks @[]) - (each pair pairs (array/push ks ((norm-node (in (vview pair) 0)) :val))) - (def srt (sorted ks)) - (when (deep= srt @[:b :g :r]) srt))) + (when (and fast (os/getenv "JOLT_SHAPE") (node :shape)) + (def sk (node :shape)) + (if (pv/pvec? sk) (pv/pv->array sk) (if (or (tuple? sk) (array? sk)) sk nil)))) (if shape-keys (do (def desc (shape-for shape-keys)) diff --git a/src/jolt/core.janet b/src/jolt/core.janet index 5095756..feba8db 100644 --- a/src/jolt/core.janet +++ b/src/jolt/core.janet @@ -495,6 +495,42 @@ (error "Vector arg to map conj must be a pair"))) result))))))))))))) +# --- shape records (hidden classes, jolt-t34) ------------------------------- +# A "shape record" is a cheap fixed-layout representation for a map literal +# whose keys are a known compile-time set (e.g. a vec3 {:r :g :b}). It is a +# Janet tuple [SHAPE v0 v1 ...] where SHAPE is an interned descriptor struct +# {:jolt/shape KEYS :idx {k->pos}}. Construction is a tuple (≈2x cheaper than a +# struct), const-keyword lookup compiles to an index, and general map ops below +# recognize it via shape-rec? and treat it as a map — so it is transparent +# wherever it flows. Created only by the backend when JOLT_SHAPE is on and the +# inference proves the shape; the runtime support here is always present so a +# shape value is handled correctly regardless. +(def- shape-cache @{}) # sorted-keys-tuple -> interned shape descriptor +(defn shape-for + "Interned shape descriptor for an ordered key vector (keys in layout order)." + [keyv] + (def kk (tuple ;keyv)) + (or (get shape-cache kk) + (let [idx @{}] + (var i 0) (each k keyv (put idx k i) (++ i)) + (def desc (struct :jolt/shape kk :idx (table/to-struct idx))) + (put shape-cache kk desc) + desc))) +(defn shape-rec? [x] + (and (tuple? x) (> (length x) 0) + (struct? (in x 0)) (not (nil? (in (in x 0) :jolt/shape))))) +(defn shape-keys [rec] ((in rec 0) :jolt/shape)) +(defn shape-get [rec k default] + (def pos (get ((in rec 0) :idx) k)) + (if (nil? pos) default (in rec (+ pos 1)))) +(defn shape-assoc [rec k v] + # assoc on a known key keeps the layout; a new key falls back to a struct + (def desc (in rec 0)) + (def pos (get (desc :idx) k)) + (if (nil? pos) + (let [t @{}] (each kk (desc :jolt/shape) (put t kk (shape-get rec kk nil))) (put t k v) (table/to-struct t)) + (let [a (array ;rec)] (put a (+ pos 1) v) (tuple ;a)))) + (defn core-assoc [m & kvs] (when (odd? (length kvs)) (error "assoc expects an even number of key/value arguments")) @@ -504,6 +540,10 @@ (and (struct? m) (get m :jolt/type))) (error (string "assoc requires a map or vector, got " (type m)))) (cond + (shape-rec? m) + (do (var result m) (var i 0) + (while (< i (length kvs)) (set result (shape-assoc result (kvs i) (kvs (+ i 1)))) (+= i 2)) + result) (core-sorted-map? m) ((sorted-op m :assoc) m kvs) (phm? m) (do (var result m) (var i 0) (while (< i (length kvs)) (set result (phm-assoc result (kvs i) (kvs (+ i 1)))) (+= i 2)) result) @@ -560,42 +600,6 @@ (do (var result @{}) (each k (keys m) (var in-ks false) (each k2 ks (if (deep= k k2) (do (set in-ks true) (break)))) (if (not in-ks) (put result k (m k)))) (if (struct? m) (table/to-struct result) result)))) -# --- shape records (hidden classes, jolt-t34) ------------------------------- -# A "shape record" is a cheap fixed-layout representation for a map literal -# whose keys are a known compile-time set (e.g. a vec3 {:r :g :b}). It is a -# Janet tuple [SHAPE v0 v1 ...] where SHAPE is an interned descriptor struct -# {:jolt/shape KEYS :idx {k->pos}}. Construction is a tuple (≈2x cheaper than a -# struct), const-keyword lookup compiles to an index, and general map ops below -# recognize it via shape-rec? and treat it as a map — so it is transparent -# wherever it flows. Created only by the backend when JOLT_SHAPE is on and the -# inference proves the shape; the runtime support here is always present so a -# shape value is handled correctly regardless. -(def- shape-cache @{}) # sorted-keys-tuple -> interned shape descriptor -(defn shape-for - "Interned shape descriptor for an ordered key vector (keys in layout order)." - [keyv] - (def kk (tuple ;keyv)) - (or (get shape-cache kk) - (let [idx @{}] - (var i 0) (each k keyv (put idx k i) (++ i)) - (def desc (struct :jolt/shape kk :idx (table/to-struct idx))) - (put shape-cache kk desc) - desc))) -(defn shape-rec? [x] - (and (tuple? x) (> (length x) 0) - (struct? (in x 0)) (not (nil? (in (in x 0) :jolt/shape))))) -(defn shape-keys [rec] ((in rec 0) :jolt/shape)) -(defn shape-get [rec k default] - (def pos (get ((in rec 0) :idx) k)) - (if (nil? pos) default (in rec (+ pos 1)))) -(defn shape-assoc [rec k v] - # assoc on a known key keeps the layout; a new key falls back to a struct - (def desc (in rec 0)) - (def pos (get (desc :idx) k)) - (if (nil? pos) - (let [t @{}] (each kk (desc :jolt/shape) (put t kk (shape-get rec kk nil))) (put t k v) (table/to-struct t)) - (let [a (array ;rec)] (put a (+ pos 1) v) (tuple ;a)))) - (defn core-get [m k &opt default] (default default nil) (if (nil? m) default