From 8192fc9541e4ee143d6ff4ac16d7622c357bcd8d Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Wed, 17 Jun 2026 02:11:38 +0000 Subject: [PATCH] Keep a nil nested in a collection used as a map key (jolt-zcm9) (#157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit canon-key canonicalizes a collection key by re-keying a native Janet table by the canonical form of each element/entry. canon-key returns nil for nil, and a Janet struct can't hold a nil key or value, so a nil set member / nil map key / nil map value was dropped during canonicalization — #{nil 1} canonicalized like #{1} and collided as a map key. So (count {#{nil 1} :a, #{1} :b}) was 1 and (get {#{nil 1} :a} #{1}) was :a. Box a nested nil before it goes into the table. The marker has to be value-hashable, not the identity-hashed mutable-table sentinel the transients use: the canonical struct becomes a long-lived phm key, and its hash must survive the marshal/snapshot/fork that init-cached relies on — an unmarshalled table gets a fresh address, so its hash isn't preserved and the map can't find its own key. An interned keyword hashes by content. Collision risk is only a real value equal to that exact keyword, the same negligible class as canon-key's existing set/map struct aliasing. The transient sentinel stays a mutable table (it's built and consumed within one op, never crossing a marshal boundary, so identity hashing is stable there). Co-authored-by: Yogthos --- src/jolt/core_types.janet | 42 +++++++++++++++++++++++++-------------- test/spec/maps-spec.janet | 14 +++++++++++++ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/jolt/core_types.janet b/src/jolt/core_types.janet index c17a20a..f65590c 100644 --- a/src/jolt/core_types.janet +++ b/src/jolt/core_types.janet @@ -73,6 +73,19 @@ (if (>= i (length t)) nil @[(in t i) (fn [] (indexed-cells t (+ i 1)))])) +# A nil element/key/value nested in a collection used AS a map key would be +# dropped when canon-key re-keys a native Janet table (a Janet struct can't hold a +# nil key or value), so #{nil 1} would canonicalize like #{1} and collide as a key +# (jolt-zcm9). Box a nested nil to a marker. It must be VALUE-hashable: the +# canonical struct becomes a long-lived phm key whose hash has to survive the +# marshal/snapshot/fork that init-cached uses, so an identity-hashed mutable table +# (like the transient sentinel below) won't do — its hash isn't preserved across +# unmarshal. An interned keyword hashes by content. Collision risk is only a real +# element equal to this exact keyword — the same negligible class as canon-key's +# existing set/map struct aliasing. +(def canon-nil (keyword "jolt.lang/canonical-nil")) +(defn- canon-box [c] (if (nil? c) canon-nil c)) + # Canonicalize a collection key/element to a value-hashable Janet struct/tuple so # the PHM/PHS treat value-equal maps/vectors as the same key (Janet hashes tables # by identity otherwise). Installed into phm via set-canonicalize-key!. @@ -82,33 +95,32 @@ (cond (pvec? k) (tuple ;(map canon-key (pv->array k))) (plist? k) (tuple ;(map canon-key (pl->array k))) - (set? k) (do (def t @{}) (each e (phs-seq k) (put t (canon-key e) true)) (table/to-struct t)) - (phm? k) (do (def t @{}) (each pair (phm-entries k) (put t (canon-key (in pair 0)) (canon-key (in pair 1)))) (table/to-struct t)) + (set? k) (do (def t @{}) (each e (phs-seq k) (put t (canon-box (canon-key e)) true)) (table/to-struct t)) + (phm? k) (do (def t @{}) (each pair (phm-entries k) (put t (canon-box (canon-key (in pair 0))) (canon-box (canon-key (in pair 1))))) (table/to-struct t)) # sorted colls canonicalize like their unsorted counterparts, so # (get {(sorted-map :a 1) :hit} {:a 1}) finds the key - (core-sorted-map? k) (do (def t @{}) (each e (sorted-entries-arr k) (put t (canon-key (vnth e 0)) (canon-key (vnth e 1)))) (table/to-struct t)) - (core-sorted-set? k) (do (def t @{}) (each x (sorted-entries-arr k) (put t (canon-key x) true)) (table/to-struct t)) + (core-sorted-map? k) (do (def t @{}) (each e (sorted-entries-arr k) (put t (canon-box (canon-key (vnth e 0))) (canon-box (canon-key (vnth e 1))))) (table/to-struct t)) + (core-sorted-set? k) (do (def t @{}) (each x (sorted-entries-arr k) (put t (canon-box (canon-key x)) true)) (table/to-struct t)) (and (table? k) (get k :jolt/deftype)) - (do (def t @{}) (each kk (keys k) (when (not= kk :jolt/deftype) (put t kk (canon-key (get k kk))))) (table/to-struct t)) - (struct? k) (do (def t @{}) (each kk (keys k) (put t (canon-key kk) (canon-key (get k kk)))) (table/to-struct t)) + (do (def t @{}) (each kk (keys k) (when (not= kk :jolt/deftype) (put t kk (canon-box (canon-key (get k kk)))))) (table/to-struct t)) + (struct? k) (do (def t @{}) (each kk (keys k) (put t (canon-box (canon-key kk)) (canon-box (canon-key (get k kk))))) (table/to-struct t)) (array? k) (tuple ;(map canon-key k)) (tuple? k) (tuple ;(map canon-key k)) k))) (set-canonicalize-key! canon-key) # Janet tables silently drop a nil key (put/get with nil is a no-op), but Clojure -# maps allow a nil key. The transient map keys its native table by canon-key, and -# canon-key returns nil only for nil input — so route nil to a unique sentinel. -# The sentinel is a fresh mutable table; canon-key never produces one, so it can't -# collide with the canon-key of any real key. (phm keeps its own has-nil slot.) +# maps allow a nil key. The transient map/set keys its native table by canon-key, +# which returns nil only for nil input — so route nil to a unique sentinel. This +# one is a fresh mutable table (canon-key never produces one, so no collision); it +# is fine here because a transient's native table is built and consumed within one +# operation and never crosses a marshal boundary, so identity hashing is stable. (def tbl-nil-key @{}) (defn tbl-key [k] (if (nil? k) tbl-nil-key (canon-key k))) -# A transient SET stores `(tbl-key x) -> x`, i.e. the member IS the table value. -# A nil member can't be a Janet table value either (put with a nil value drops -# the entry), so box nil as the same sentinel and unbox on read-back. The -# sentinel is a fresh table canon-key never produces, so it can't collide with a -# real member. +# A transient SET stores `(tbl-key x) -> x`, i.e. the member IS the table value. A +# nil member can't be a Janet table value either (put with a nil value drops the +# entry), so box nil as the same sentinel and unbox on read-back. (defn tbl-box [x] (if (nil? x) tbl-nil-key x)) (defn tbl-unbox [v] (if (= v tbl-nil-key) nil v)) diff --git a/test/spec/maps-spec.janet b/test/spec/maps-spec.janet index 706d75f..9ebd5e2 100644 --- a/test/spec/maps-spec.janet +++ b/test/spec/maps-spec.janet @@ -72,6 +72,20 @@ ["frequencies of maps" "2" "(get (frequencies [{:a 1} (hash-map :a 1)]) {:a 1})"] ["group-by collection key" "1" "(count (group-by identity [{:a 1} (hash-map :a 1)]))"]) +# A nil element/key/value INSIDE a collection used as a map key must survive key +# canonicalization — it used to be dropped, so the nil-containing collection +# collided with the nil-free one (jolt-zcm9). +(defspec "map / nil inside a collection key (jolt-zcm9)" + ["set key w/ nil distinct" "2" "(count {#{nil 1} :a, #{1} :b})"] + ["set key w/ nil neg lookup" "nil" "(get {#{nil 1} :a} #{1})"] + ["set key w/ nil pos lookup" ":a" "(get {#{nil 1} :a} #{nil 1})"] + ["set key just nil distinct" "false" "(= {#{nil} :x} {#{} :x})"] + ["map nil-value key distinct" "2" "(count {{:a nil} 1, {} 2})"] + ["map nil-value key neg" "nil" "(get {{:a nil} 1} {})"] + ["map nil-value key pos" "1" "(get {{:a nil} 1} {:a nil})"] + ["map nil-key key distinct" "2" "(count {{nil :a} 1, {} 2})"] + ["map nil-key key pos" "1" "(get {{nil :a} 1} {nil :a})"]) + # Strictness: assoc bounds-checks vector indices; dissoc requires a map; # count rejects scalars; numerator/denominator have no ratio type. (defspec "map / strictness (throws like Clojure)"