From c13a8ee402132ea667fff95650bc57c410c82951 Mon Sep 17 00:00:00 2001 From: Dmitri Sotnikov Date: Tue, 16 Jun 2026 04:41:49 +0000 Subject: [PATCH] Add benchmark suite for alloc/dispatch/collection workloads (jolt-1r86) (#135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ray tracer is float-compute-bound (devirt, alloc removal, type-proving all measured flat on it), so it can't validate the optimization passes. Add a small cross-language suite (AWFY + CLBG style, portable Clojure) isolating the axes it misses: binary-trees allocation / GC pressure (escaping short-lived records) dispatch megamorphic protocol dispatch (~1M dispatches/s; WP can't devirt) collections persistent map/vector churn bench/run.sh runs them; bench/README.md maps each to the pass it exercises. collections immediately surfaced jolt-684u: the persistent hash map is O(n) per assoc (flat copy-on-write bucket array, not a HAMT) — n=4000 assocs take 50s. Invisible to the ray tracer (no maps). Co-authored-by: Yogthos --- bench/README.md | 48 ++++++++++++++++++++++++++++++++++++ bench/binary_trees.clj | 54 ++++++++++++++++++++++++++++++++++++++++ bench/collections.clj | 46 ++++++++++++++++++++++++++++++++++ bench/dispatch.clj | 56 ++++++++++++++++++++++++++++++++++++++++++ bench/run.sh | 39 +++++++++++++++++++++++++++++ 5 files changed, 243 insertions(+) create mode 100644 bench/README.md create mode 100644 bench/binary_trees.clj create mode 100644 bench/collections.clj create mode 100644 bench/dispatch.clj create mode 100755 bench/run.sh diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..6ecfba0 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,48 @@ +# jolt benchmark suite + +Benchmarks that isolate the workload axes jolt's optimizing passes target. The +ray tracer (`examples/ray-tracer`) is **float-compute-bound** — its time is +irreducible algorithmic math (hit-testing + transcendentals), and devirt, +allocation removal, and type-proving all measured **flat** on it. So it can't +tell us whether those passes work. These benchmarks make each pass's target +workload the *dominant* cost. + +Reference: the cross-language suites these draw from — +[Are We Fast Yet?](https://github.com/smarr/are-we-fast-yet) (Marr et al., DLS '16) +and the [Computer Language Benchmarks Game](https://benchmarksgame-team.pages.debian.net/benchmarksgame/). +The benchmarks are portable Clojure, so they also run on JVM Clojure for an +absolute reference. + +## Benchmarks + +| Benchmark | Axis | Pass it exercises | Source | +|---|---|---|---| +| `binary-trees` | allocation / GC pressure (escaping short-lived records) | jolt-15jq scalar-replace, jolt-8flj escape analysis | CLBG | +| `dispatch` | polymorphic (megamorphic) protocol dispatch | jolt-41m devirt, inline-cache | AWFY-style | +| `collections` | persistent map/vector churn (32-way tries) | persistent structures, transients | CLBG k-nucleotide-style | + +What the ray tracer does **not** capture and these do: allocation as the +bottleneck (~7% there), megamorphic dispatch (its dispatch is monomorphic and +cheap), and persistent-collection throughput (it uses fixed records, no +collections in the hot loop). + +Planned additions: Richards / DeltaBlue (heavier OO dispatch), a **monomorphic** +dispatch variant (where devirt *can* fire — the megamorphic `dispatch` can't), +NBody (float control, parallels the ray tracer), k-nucleotide proper. + +## Running + +```sh +jpm build && export PATH="$PWD/build:$PATH" +bench/run.sh # whole-program optimization on (default) +JOLT_WHOLE_PROGRAM=0 bench/run.sh # WP off, to measure what WP buys +bench/run.sh binary-trees 16 # one benchmark, custom size +``` + +## A/B against a change + +To measure a pass, run the suite on `main`, then on the branch, back to back +(same machine, quiet) — the protocol used for `test/bench/core-bench.janet` and +the ray tracer. Each benchmark prints `runs: [...]` and `mean: N ms`; compare +the means. A pass is worth landing when it moves a benchmark whose axis it +targets, even if the ray tracer stays flat. diff --git a/bench/binary_trees.clj b/bench/binary_trees.clj new file mode 100644 index 0000000..88079a8 --- /dev/null +++ b/bench/binary_trees.clj @@ -0,0 +1,54 @@ +;; binary-trees (Computer Language Benchmarks Game) — an ALLOCATION/GC stress +;; test. Builds and discards millions of short-lived `Node` records; the nodes +;; ESCAPE (stored in the tree, walked later), so this is the regime jolt-8flj +;; (escape analysis) targets and the ray tracer never exercises (~7% alloc). +;; +;; Portable Clojure: runs on jolt and JVM Clojure for cross-impl comparison. +;; jolt -m binary-trees 14 (JOLT_DIRECT_LINK=1 JOLT_WHOLE_PROGRAM=1) +;; clojure -M -m binary-trees 14 +(ns binary-trees) + +(defrecord Node [left right]) + +(defn make-tree [depth] + (if (zero? depth) + (->Node nil nil) + (->Node (make-tree (dec depth)) (make-tree (dec depth))))) + +(defn check-tree [node] + (let [l (:left node)] + (if (nil? l) + 1 + (+ (+ 1 (check-tree l)) (check-tree (:right node)))))) + +(defn run [max-depth] + (let [min-depth 4 + stretch-depth (inc max-depth) + _ (check-tree (make-tree stretch-depth)) + long-lived (make-tree max-depth)] + (loop [d min-depth acc 0] + (if (<= d max-depth) + (let [iterations (bit-shift-left 1 (+ (- max-depth d) min-depth)) + sum (loop [i 0 s 0] + (if (< i iterations) + (recur (inc i) (+ s (check-tree (make-tree d)))) + s))] + (recur (+ d 2) (+ acc sum))) + ;; touch the long-lived tree so it isn't dead-code-eliminated + (+ acc (check-tree long-lived)))))) + +(defn -main [& args] + (let [max-depth (if (seq args) (Integer/parseInt (first args)) 14)] + (dotimes [_ 2] (run (min max-depth 10))) ; warmup + (let [runs 3 + times (mapv (fn [_] + (let [t0 (System/nanoTime) + r (run max-depth) + ms (/ (- (System/nanoTime) t0) 1000000.0)] + [ms r])) + (range runs)) + mss (mapv first times) + mean (/ (reduce + mss) runs)] + (println "binary-trees depth" max-depth "checksum" (second (first times))) + (println "runs:" (mapv (fn [t] (/ (Math/round (* t 10.0)) 10.0)) mss)) + (println "mean:" (/ (Math/round (* mean 10.0)) 10.0) "ms")))) diff --git a/bench/collections.clj b/bench/collections.clj new file mode 100644 index 0000000..16e9807 --- /dev/null +++ b/bench/collections.clj @@ -0,0 +1,46 @@ +;; collections — PERSISTENT-COLLECTION churn. Builds and reads persistent maps +;; and vectors (32-way hash/array tries) under heavy assoc/update/conj/lookup, a +;; word-count-style workload (cf. CLBG k-nucleotide). Exercises jolt's persistent +;; data structures and (eventually) transients — an axis the ray tracer (fixed +;; records, no collections in the hot loop) doesn't touch. +;; +;; Portable Clojure (jolt + JVM Clojure). +;; jolt -m collections 200000 (JOLT_DIRECT_LINK=1 JOLT_WHOLE_PROGRAM=1) +(ns collections) + +;; map churn: accumulate a frequency map over a stream of keys, then sum it back +(defn freq-map [n buckets] + (loop [i 0 m {}] + (if (< i n) + (recur (inc i) + (let [k (mod (* i 2654435761) buckets)] + (assoc m k (+ 1 (get m k 0))))) + m))) + +(defn sum-vals [m] + (reduce (fn [acc k] (+ acc (get m k))) 0 (keys m))) + +;; vector churn: conj many, then reduce +(defn vec-sum [n] + (let [v (loop [i 0 v []] (if (< i n) (recur (inc i) (conj v (mod i 1000))) v))] + (reduce + 0 v))) + +(defn run [n] + (let [m (freq-map n 4096)] + (+ (sum-vals m) (vec-sum (quot n 4))))) + +(defn -main [& args] + (let [n (if (seq args) (Integer/parseInt (first args)) 200000)] + (dotimes [_ 2] (run (quot n 4))) ; warmup + (let [runs 3 + times (mapv (fn [_] + (let [t0 (System/nanoTime) + r (run n) + ms (/ (- (System/nanoTime) t0) 1000000.0)] + [ms r])) + (range runs)) + mss (mapv first times) + mean (/ (reduce + mss) runs)] + (println "collections n" n "result" (second (first times))) + (println "runs:" (mapv (fn [t] (/ (Math/round (* t 10.0)) 10.0)) mss)) + (println "mean:" (/ (Math/round (* mean 10.0)) 10.0) "ms")))) diff --git a/bench/dispatch.clj b/bench/dispatch.clj new file mode 100644 index 0000000..a6a2410 --- /dev/null +++ b/bench/dispatch.clj @@ -0,0 +1,56 @@ +;; dispatch — a POLYMORPHIC-DISPATCH stress test. A protocol method is called in +;; a hot loop over a heterogeneous (megamorphic) collection of record types, with +;; minimal per-call work, so protocol dispatch dominates. This is the regime +;; jolt-41m (devirtualization) and the inline-cache target, and the one the ray +;; tracer can't reveal — its dispatch is monomorphic and a small fraction of the +;; float-math cost (devirt measured FLAT there). +;; +;; Portable Clojure (jolt + JVM Clojure). +;; jolt -m dispatch 20000 (JOLT_DIRECT_LINK=1 JOLT_WHOLE_PROGRAM=1) +(ns dispatch) + +(defprotocol Shape + (area [s]) + (sides [s])) + +(defrecord Circle [r] Shape (area [_] (* (* 3.14159 r) r)) (sides [_] 0)) +(defrecord Square [s] Shape (area [_] (* s s)) (sides [_] 4)) +(defrecord Triangle [b h] Shape (area [_] (* (* 0.5 b) h)) (sides [_] 3)) +(defrecord Rect [w h] Shape (area [_] (* w h)) (sides [_] 4)) + +(defn build-shapes [n] + (mapv (fn [i] + (let [k (mod i 4)] + (cond + (= k 0) (->Circle (+ 1 (mod i 7))) + (= k 1) (->Square (+ 1 (mod i 5))) + (= k 2) (->Triangle (+ 1 (mod i 3)) (+ 2 (mod i 6))) + :else (->Rect (+ 1 (mod i 4)) (+ 1 (mod i 8)))))) + (range n))) + +;; megamorphic: every element may be a different type -> the call site sees all 4 +(defn sum-area [shapes] + (reduce (fn [acc s] (+ (+ acc (area s)) (sides s))) 0.0 shapes)) + +(defn run [iters] + (let [shapes (build-shapes 1000)] + (loop [i 0 acc 0.0] + (if (< i iters) + (recur (inc i) (+ acc (sum-area shapes))) + acc)))) + +(defn -main [& args] + (let [iters (if (seq args) (Integer/parseInt (first args)) 20000)] + (dotimes [_ 2] (run (quot iters 4))) ; warmup + (let [runs 3 + times (mapv (fn [_] + (let [t0 (System/nanoTime) + r (run iters) + ms (/ (- (System/nanoTime) t0) 1000000.0)] + [ms r])) + (range runs)) + mss (mapv first times) + mean (/ (reduce + mss) runs)] + (println "dispatch iters" iters "result" (second (first times))) + (println "runs:" (mapv (fn [t] (/ (Math/round (* t 10.0)) 10.0)) mss)) + (println "mean:" (/ (Math/round (* mean 10.0)) 10.0) "ms")))) diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..ade7a09 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Run the jolt benchmark suite and print mean ms per benchmark. +# +# Each benchmark isolates an axis the ray tracer (float-compute-bound) doesn't +# capture — see README.md. Run back-to-back against `main` to measure a pass's +# impact (the same protocol as test/bench/core-bench.janet). +# +# bench/run.sh # default sizes, whole-program optimization on +# JOLT_WHOLE_PROGRAM=0 bench/run.sh # compare with WP off +# bench/run.sh binary-trees # one benchmark +# +# Needs `jolt` on PATH (build with `jpm build`; export PATH="$PWD/build:$PATH"). +set -e +cd "$(dirname "$0")" + +export JOLT_DIRECT_LINK="${JOLT_DIRECT_LINK:-1}" +export JOLT_WHOLE_PROGRAM="${JOLT_WHOLE_PROGRAM:-1}" +export JOLT_APP_PATHS="$PWD" +export JOLT_PATH="$PWD" + +# name:default-arg (arg sized to run in a few seconds each) +# NOTE collections is small because the persistent map is O(n)/assoc (jolt-684u); +# raise it once that's fixed to a HAMT. +BENCHES="binary-trees:14 dispatch:2000 collections:1500" + +run_one() { + ns="${1%%:*}"; arg="${2:-${1##*:}}" + printf '%-16s ' "$ns" + jolt -m "$ns" "$arg" 2>&1 | awk '/^mean:/{print}' +} + +if [ -n "$1" ]; then + for spec in $BENCHES; do + [ "${spec%%:*}" = "$1" ] && run_one "$spec" "$2" + done +else + echo "jolt benchmark suite (WP=$JOLT_WHOLE_PROGRAM)" + for spec in $BENCHES; do run_one "$spec"; done +fi