From 597f76a7538ab949f666aaabf36ba9e6077c5cd7 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Mon, 8 Jun 2026 19:13:30 -0400 Subject: [PATCH] test/ci: make clojure-test-suite per-file deadline configurable (JOLT_SUITE_TIMEOUT) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 6s per-file deadline + baseline=3981 were tuned to a fast dev machine; a slower CI runner could time out a sub-second finite file, dropping total-pass below baseline and flaking CI red. The deadline is now an env var (default 6 locally); CI sets it to 20s for headroom. Characterized the 6 timeouts: identical pass/timeout counts at 6s and 20s, so all 6 are genuinely-infinite hangs (killed at any deadline) and the 227 finite files finish well under 6s. So no baseline margin is needed — a generous CI deadline removes the flake risk while preserving full regression sensitivity. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/tests.yml | 5 +++++ test/integration/clojure-test-suite-test.janet | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3bf6f19..0e712bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,11 @@ jobs: runs-on: ubuntu-latest env: JANET_VERSION: v1.41.2 # bump to match the version Jolt is developed against + # Per-file deadline for the clojure-test-suite battery. Finite files finish + # in well under 1s; the genuinely-infinite ones get killed at any deadline. + # A generous value gives slow CI runners headroom so a sub-second file + # spiking doesn't time out and drop total-pass below the baseline. + JOLT_SUITE_TIMEOUT: "20" steps: - uses: actions/checkout@v4 with: diff --git a/test/integration/clojure-test-suite-test.janet b/test/integration/clojure-test-suite-test.janet index fce50c2..07bf4ba 100644 --- a/test/integration/clojure-test-suite-test.janet +++ b/test/integration/clojure-test-suite-test.janet @@ -41,9 +41,14 @@ (def baseline-pass 3981) # A file is "clean" when it ran with zero failures AND zero errors. (def baseline-clean-files 66) -# Per-file wall-clock budget (seconds). Normal files finish in well under 1s; -# this only fires on infinite-sequence hangs. -(def per-file-timeout 6) +# Per-file wall-clock budget (seconds). Normal files finish in well under 1s, so +# this normally only fires on genuinely-infinite-sequence hangs. It's an env var +# (JOLT_SUITE_TIMEOUT) so CI — whose runners are slower than a dev machine — can +# give slow-but-finite files generous headroom without timing them out (which +# would drop total-pass below the baseline and flake CI red). Default 6 locally. +(def per-file-timeout + (let [e (os/getenv "JOLT_SUITE_TIMEOUT")] + (or (and e (scan-number e)) 6))) (defn- walk [dir acc] (each e (os/dir dir)