From 9ab36eb97f618238d5d12e2ca3a9af4a3f44487a Mon Sep 17 00:00:00 2001 From: Yogthos Date: Wed, 10 Jun 2026 23:22:36 -0400 Subject: [PATCH] deps: global gitlibs-style clone cache + :tasks runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git clones now default to a shared, sha-immutable cache — $JOLT_GITLIBS, else /gitlibs — instead of a per-project ./jpm_tree, the tools.gitlibs ~/.gitlibs model. Passing tree explicitly still works (tests do). The resolved-roots cache moves out of the clone tree to the project-local .cpcache/jolt-deps.jdn, since roots depend on the project while clones don't. deps.edn grows :tasks, the honest subset of babashka's: a string task is a shell command, a map task is {:main-opts [...] :doc}. jolt-deps tasks lists them (merged user+project), jolt-deps task NAME runs one. Bare-expression tasks are out of scope: the reader hands back parsed data and round-tripping to source is fragile. Also fixes load-config skipping the symbol-key normalization when only one config file existed — :tasks/:deps keys stayed raw reader symbols (which embed positions and never compare equal), so lookups missed. Regression rows in deps-tasks-test; docs updated for the whole tools.deps surface (aliases, -A/-M, user config, conflicts, gitlibs cache, tasks). --- docs/building-and-deps.md | 27 ++++++- docs/tools-deps.md | 26 +++++- src/jolt/deps.janet | 105 +++++++++++++++++++------ src/jolt/deps_cli.janet | 16 ++++ test/integration/deps-tasks-test.janet | 89 +++++++++++++++++++++ 5 files changed, 231 insertions(+), 32 deletions(-) create mode 100644 test/integration/deps-tasks-test.janet diff --git a/docs/building-and-deps.md b/docs/building-and-deps.md index fa6a499..c3023eb 100644 --- a/docs/building-and-deps.md +++ b/docs/building-and-deps.md @@ -63,6 +63,10 @@ jolt-deps path # print the resolved roots (':'-joined) jolt-deps run FILE [args] # resolve, then run `jolt FILE …` jolt-deps repl # resolve, then start a REPL jolt-deps -e EXPR [args] # resolve, then evaluate EXPR +jolt-deps -A:dev path # include the :dev alias's extra paths/deps +jolt-deps -M:test [args] # run the :test alias's :main-opts through jolt +jolt-deps tasks # list :tasks from deps.edn +jolt-deps task NAME [args] # run a task ``` `jolt-deps` launches the `jolt` binary it finds on `PATH` (override with @@ -88,10 +92,27 @@ jolt-deps run -m myapp.main dependency's own `deps.edn` are resolved too. - **local deps** — `{:local/root "../path"}`. - The project's own `:paths` (default `["src"]`) are included. +- **aliases** — `:aliases {:dev {:extra-paths ["dev"] :extra-deps {…} + :main-opts ["-e" "…"]}}`, selected with `-A:dev` (or several: `-A:dev:test`). + `:extra-paths`/`:extra-deps` accumulate across selected aliases; + `:main-opts` is last-wins and runs via `-M:alias`. +- **user config** — a `deps.edn` under `$JOLT_CONFIG` (else + `$XDG_CONFIG_HOME/jolt`, else `~/.jolt`) merges beneath the project's, the + way `~/.clojure/deps.edn` does: `:deps`/`:aliases`/`:tasks` merge per key + with the project winning. +- **tasks** — `:tasks {clean "rm -rf target" test {:doc "run the suite" + :main-opts ["-e" "(run-tests)"]}}`. A string task is a shell command; a map + task runs jolt with its `:main-opts`. `jolt-deps tasks` lists, `jolt-deps + task NAME` runs. -Resolution reuses jpm's git fetch and cache (a dependency is cloned once into -`jpm_tree/.cache` and reused). Resolved roots are cached on a hash of `deps.edn`, -so an unchanged `deps.edn` doesn't re-fetch. +Conflicts resolve the tools.deps way: resolution is breadth-first, so a +top-level coordinate always beats a transitive one for the same lib, and +conflicting coordinates print a warning naming both. + +Git clones land in a global, sha-immutable cache shared across projects — +`$JOLT_GITLIBS`, else `/gitlibs` (the `~/.gitlibs` model). The +resolved roots are cached per project in `.cpcache/jolt-deps.jdn`, keyed on a +hash of the project `deps.edn` + the user `deps.edn` + the selected aliases. ### What's not diff --git a/docs/tools-deps.md b/docs/tools-deps.md index 77f1e20..ff074bb 100644 --- a/docs/tools-deps.md +++ b/docs/tools-deps.md @@ -42,11 +42,29 @@ syntax overlap for the `:deps`/`:paths` subset), then walks `:deps`: - `:mvn/*` and anything else → ignored. Each resolved dependency contributes its own `:paths` (default `["src"]`) as -source roots, and we recurse into its `deps.edn` for transitive deps. The result +source roots; the walk is **breadth-first** so every top-level coordinate +registers before any transitive one — a top-level pin always wins, matching +tools.deps, and a coordinate conflict warns on stderr naming both. The result is a de-duplicated, ordered list of directories. `resolve-deps-cached` memoizes -that list in the tree keyed on a hash of `deps.edn`, so an unchanged file doesn't -re-fetch. jpm is loaded lazily (`require`, not `import`) so it's pulled in only -when resolving — never embedded in a built binary. +that list in the project-local `.cpcache/jolt-deps.jdn`, keyed on a hash of the +project `deps.edn` + the user-level `deps.edn` + the selected aliases. jpm is +loaded lazily (`require`, not `import`) so it's pulled in only when resolving — +never embedded in a built binary. + +Three tools.deps features are mirrored in reduced form. **Aliases**: `:aliases` +entries supply `:extra-paths`/`:extra-deps` (accumulate across the aliases +selected with `-A:a:b`) and `:main-opts` (last-wins, run with `-M:alias`). +**User config**: a `deps.edn` under `$JOLT_CONFIG` (else +`$XDG_CONFIG_HOME/jolt`, else `~/.jolt`) merges beneath the project file, +per key, project wins. **Tasks**: the honest subset of babashka's — a string +task is a shell command, a map task is `{:main-opts […] :doc "…"}`; bare +Clojure expressions aren't supported because the reader hands back parsed +data, and round-tripping it to source isn't worth the fragility. + +Clones default to a global sha-immutable cache (`$JOLT_GITLIBS`, else +`/gitlibs`) shared across projects, the `tools.gitlibs` +`~/.gitlibs` model; per-project trees remain available by passing `tree` +explicitly. The loader (`evaluator.janet/find-ns-file`) resolves a namespace by searching the context's `:source-paths` in order (the stdlib `src/jolt` first), trying `.clj` diff --git a/src/jolt/deps.janet b/src/jolt/deps.janet index 669b138..aef72f3 100644 --- a/src/jolt/deps.janet +++ b/src/jolt/deps.janet @@ -58,6 +58,35 @@ (each m [a b] (when (dictionary? m) (eachp [k v] m (put out k v)))) out) +# Reader symbols carry position metadata, so any map keyed by SYMBOLS (deps +# libs, task names) must be re-keyed by name before merging or deduping. +(defn- sym-name [x] + (if (and (dictionary? x) (get x :name)) + (if-let [ns (get x :ns)] + (string ns "/" (get x :name)) + (get x :name)) + (string x))) + +(defn- merge-by-name [a b] # union of symbol-keyed dictionaries, b wins + (def out @{}) + (each m [a b] (when (dictionary? m) (eachp [k v] m (put out (sym-name k) v)))) + out) + +(defn- mkdirs [p] + (def abs (string/has-prefix? "/" p)) + (var acc nil) + (each seg (filter |(not= "" $) (string/split "/" p)) + (set acc (cond (nil? acc) (if abs (string "/" seg) seg) (string acc "/" seg))) + (unless (os/stat acc) (os/mkdir acc)))) + +(defn- default-tree + "Where git clones land when no tree is given: $JOLT_GITLIBS, else + (config-dir)/gitlibs — a global, sha-immutable cache shared across projects + (the tools.gitlibs ~/.gitlibs model), not a per-project ./jpm_tree." + [] + (def g (os/getenv "JOLT_GITLIBS")) + (if (and g (> (length g) 0)) g (string (config-dir) "/gitlibs"))) + (defn load-config "The project deps.edn merged over the user-level one (config-dir)/deps.edn. tools.deps merge semantics: scalar keys and :paths replace (project wins), @@ -67,12 +96,16 @@ [deps-edn-path] (def proj (read-edn deps-edn-path)) (def user (read-edn (string (config-dir) "/deps.edn"))) - (cond - (nil? user) proj - (nil? proj) user - (let [out (merge-per-key user proj)] - (put out :deps (merge-per-key (get user :deps) (get proj :deps))) - (put out :aliases (merge-per-key (get user :aliases) (get proj :aliases))) + (if (and (nil? user) (nil? proj)) + nil + # normalize even when only one file exists: :deps/:tasks come back keyed + # by NAME (reader symbols carry position metadata and never compare equal) + (let [u (when (dictionary? user) user) + p (when (dictionary? proj) proj) + out (merge-per-key u p)] + (put out :deps (merge-by-name (and u (get u :deps)) (and p (get p :deps)))) + (put out :aliases (merge-per-key (and u (get u :aliases)) (and p (get p :aliases)))) + (put out :tasks (merge-by-name (and u (get u :tasks)) (and p (get p :tasks)))) out))) (defn combine-aliases @@ -107,22 +140,14 @@ `aliases` (keywords) pull :extra-paths/:extra-deps from the merged config's :aliases. The user-level deps.edn (see load-config) merges under the project." [deps-edn-path &opt tree aliases] - (default tree (string (os/cwd) "/jpm_tree")) - (os/mkdir tree) + (default tree (default-tree)) + (mkdirs tree) (ensure-jpm-config tree) (def edn (load-config deps-edn-path)) (def extra (combine-aliases edn aliases)) (def roots @[]) (def seen @{}) # lib name -> chosen coordinate (for conflict reporting) (defn add-root [r] (unless (index-of r roots) (array/push roots r))) - # Reader symbols carry position metadata, so dedup/conflict keys must use the - # NAME, never (string lib) — two my/b symbols from different files differ. - (defn lib-name [lib] - (if (and (dictionary? lib) (get lib :name)) - (if-let [ns (get lib :ns)] - (string ns "/" (get lib :name)) - (get lib :name)) - (string lib))) (defn coord-str [spec] (cond (and (dictionary? spec) (get spec :local/root)) @@ -137,7 +162,7 @@ (deep= (get a :git/tag) (get b :git/tag)))) (def queue @[]) (defn discover [lib spec base-dir] - (def k (lib-name lib)) + (def k (sym-name lib)) (if-let [prev (get seen k)] (unless (coord= prev spec) (eprintf "WARNING: %s: conflicting coordinates — using %s, ignoring %s" @@ -158,13 +183,13 @@ # the project's own paths (+ alias extra paths) lead the roots (each r (src-roots (os/cwd) edn) (add-root r)) (each pp (extra :extra-paths) (add-root (string (os/cwd) "/" pp))) - # breadth-first: every top-level dep (incl. alias :extra-deps) registers - # before any transitive dep — so a top-level coordinate always wins, - # matching tools.deps - (eachp [lib spec] (or (and (dictionary? edn) (get edn :deps)) {}) - (discover lib spec (os/cwd))) + # breadth-first: every top-level dep registers before any transitive dep — + # so a top-level coordinate always wins, matching tools.deps. Alias + # :extra-deps go first: a selected alias's pin beats the project's. (eachp [lib spec] (extra :extra-deps) (discover lib spec (os/cwd))) + (eachp [lib spec] (or (and (dictionary? edn) (get edn :deps)) {}) + (discover lib spec (os/cwd))) (while (> (length queue) 0) (def [dep-edn dir] (get queue 0)) (array/remove queue 0) @@ -178,10 +203,12 @@ of the project deps.edn + the user deps.edn + the selected aliases, so an unchanged config resolves without re-fetching." [deps-edn-path &opt tree aliases] - (default tree (string (os/cwd) "/jpm_tree")) + (default tree (default-tree)) (when (os/stat deps-edn-path) - (os/mkdir tree) - (def cache-file (string tree "/.jolt-deps-roots.jdn")) + # the roots depend on the PROJECT (config + aliases), so their cache is + # project-local like tools.deps' .cpcache; the clone tree stays global + (os/mkdir ".cpcache") + (def cache-file ".cpcache/jolt-deps.jdn") (def user-path (string (config-dir) "/deps.edn")) (def h (hash [(slurp deps-edn-path) (when (os/stat user-path) (slurp user-path)) @@ -192,3 +219,31 @@ (let [roots (resolve-deps deps-edn-path tree aliases)] (spit cache-file (string/format "%j" {:hash h :roots roots})) roots)))) + +# --- :tasks (the honest subset of babashka's) ---------------------------------- +# A STRING task is a shell command. A MAP task carries :main-opts (jolt args — +# `-e "(...)"` covers expression tasks) and an optional :doc. Babashka-style +# bare-expression tasks aren't supported: the reader hands us parsed data, and +# round-tripping it back to source isn't worth the fragility. + +(defn tasks + "Sorted [name doc] pairs from the merged user+project :tasks." + [deps-edn-path] + (def m (get (load-config deps-edn-path) :tasks)) + (def names (sort (keys (or m @{})))) + (map (fn [n] + (def v (get m n)) + [n (when (dictionary? v) (get v :doc))]) + names)) + +(defn task-spec + "What running task `name` means: {:type :shell :cmd s} or + {:type :jolt :argv [...]}; nil when undefined." + [deps-edn-path name] + (def v (get (or (get (load-config deps-edn-path) :tasks) @{}) name)) + (cond + (nil? v) nil + (or (string? v) (buffer? v)) {:type :shell :cmd (string v)} + (and (dictionary? v) (get v :main-opts)) + {:type :jolt :argv (tuple ;(map string (get v :main-opts)))} + (error (string "task " name ": use a shell string or {:main-opts [...]}")))) diff --git a/src/jolt/deps_cli.janet b/src/jolt/deps_cli.janet index 8a64478..28d9834 100644 --- a/src/jolt/deps_cli.janet +++ b/src/jolt/deps_cli.janet @@ -14,6 +14,10 @@ # jolt-deps -M:a[:b] [args] resolve with the aliases, then run jolt with # the last alias's :main-opts ++ args # jolt-deps uberscript OUT -m NS resolve, then bundle NS + deps into one .clj +# jolt-deps tasks list :tasks (merged user+project deps.edn) +# jolt-deps task NAME [args] run a task: a string task is a shell command +# (args appended); a {:main-opts [...]} task +# runs jolt with those args ++ extra args # # -A:dev:test selects aliases (tools.deps style): their :extra-paths and # :extra-deps join the resolution. A user-level deps.edn ($JOLT_CONFIG, else @@ -60,6 +64,18 @@ (os/exit 1)))] (os/exit (exec-jolt als [;mo ;(tuple/slice argv 1)]))) (= cmd "path") (print (string/join (roots aliases) ":")) + (= cmd "tasks") + (each row (deps/tasks "deps.edn") + (print (row 0) (if (row 1) (string "\t" (row 1)) ""))) + (= cmd "task") + (let [name (get argv 1) + spec (when name (deps/task-spec "deps.edn" name))] + (cond + (nil? name) (do (eprint "jolt-deps: task needs a name") (os/exit 1)) + (nil? spec) (do (eprint "jolt-deps: no such task: " name) (os/exit 1)) + (= :shell (spec :type)) + (os/exit (os/execute ["sh" "-c" (string/join [(spec :cmd) ;(tuple/slice argv 2)] " ")] :p)) + (os/exit (exec-jolt aliases [;(spec :argv) ;(tuple/slice argv 2)])))) (= cmd "run") (os/exit (exec-jolt aliases (tuple/slice argv 1))) (= cmd "repl") (os/exit (exec-jolt aliases [])) (= cmd "-e") (os/exit (exec-jolt aliases argv)) diff --git a/test/integration/deps-tasks-test.janet b/test/integration/deps-tasks-test.janet new file mode 100644 index 0000000..d45fc02 --- /dev/null +++ b/test/integration/deps-tasks-test.janet @@ -0,0 +1,89 @@ +# deps.edn :tasks (jolt-x4o) + the global gitlibs-style clone cache default +# (jolt-xkd). Tasks are the honest subset of babashka's: a STRING task is a +# shell command; a MAP task carries :main-opts (jolt args) and optional :doc. +# Local-only: no network, no jolt binary — the CLI wrappers stay thin. + +(import ../../src/jolt/deps :as deps) + +(def base (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-deps-tasks-" (os/time))) +(defn rmrf [p] + (when (os/stat p) + (if (= :directory (os/stat p :mode)) + (do (each e (os/dir p) (rmrf (string p "/" e))) (os/rmdir p)) + (os/rm p)))) +(rmrf base) +(defn mkdirs [p] + (def abs (string/has-prefix? "/" p)) + (var acc nil) + (each seg (filter |(not= "" $) (string/split "/" p)) + (set acc (cond (nil? acc) (if abs (string "/" seg) seg) (string acc "/" seg))) + (unless (os/stat acc) (os/mkdir acc)))) + +(each d ["proj/src" "config"] (mkdirs (string base "/" d))) +(spit (string base "/proj/deps.edn") + `{:paths ["src"] + :tasks {clean "rm -rf target" + test {:doc "run the suite" :main-opts ["-e" "(run-tests)"]} + fmt {:main-opts ["-e" "(fmt)"]}}}`) +# user-level task, and a project-shadowed name +(spit (string base "/config/deps.edn") + `{:tasks {lint "echo lint" clean "echo SHADOWED"}}`) + +(var fails 0) +(defn check [label got expected] + (if (= got expected) (print " ok " label) + (do (++ fails) (printf " FAIL %s: want %q, got %q" label expected got)))) + +(os/cd (string base "/proj")) +(os/setenv "JOLT_CONFIG" (string base "/config")) + +# --- task listing (merged user+project, sorted) --------------------------------- +(def listing (deps/tasks "deps.edn")) +(check "lists all task names" + (tuple ;(sort (map first listing))) ["clean" "fmt" "lint" "test"]) +(check "doc shows in listing" + (truthy? (some |(and (= (first $) "test") (= (get $ 1) "run the suite")) listing)) + true) + +# --- task lookup ------------------------------------------------------------------ +(check "string task is shell" + (deps/task-spec "deps.edn" "clean") {:type :shell :cmd "rm -rf target"}) +(check "project task shadows user task" + (get (deps/task-spec "deps.edn" "clean") :cmd) "rm -rf target") +(check "map task is jolt argv" + (deps/task-spec "deps.edn" "test") {:type :jolt :argv ["-e" "(run-tests)"]}) +(check "user-level task visible" + (deps/task-spec "deps.edn" "lint") {:type :shell :cmd "echo lint"}) +(check "unknown task is nil" + (deps/task-spec "deps.edn" "nope") nil) + +# --- tasks resolve with NO user config (regression: load-config skipped the +# name re-keying when only the project file existed) ----------------------------- +(os/setenv "JOLT_CONFIG" (string base "/no-such-dir")) +(check "task works without user config" + (deps/task-spec "deps.edn" "clean") {:type :shell :cmd "rm -rf target"}) +(check "listing works without user config" + (tuple ;(sort (map first (deps/tasks "deps.edn")))) ["clean" "fmt" "test"]) +(os/setenv "JOLT_CONFIG" (string base "/config")) + +# --- global clone-tree default (jolt-xkd) ---------------------------------------- +# resolution with :local deps never clones, but the default tree dir must come +# from $JOLT_GITLIBS (else (config-dir)/gitlibs), not ./jpm_tree +(os/setenv "JOLT_GITLIBS" (string base "/deep/nested/gitlibs")) +(deps/resolve-deps "deps.edn") +(check "JOLT_GITLIBS dir created (parents too)" + (truthy? (os/stat (string base "/deep/nested/gitlibs"))) true) +(check "no per-project jpm_tree" (os/stat (string base "/proj/jpm_tree")) nil) + +# roots cache is project-local .cpcache, not inside the clone tree +(deps/resolve-deps-cached "deps.edn") +(check "roots cache in ./.cpcache" + (truthy? (os/stat (string base "/proj/.cpcache/jolt-deps.jdn"))) true) + +(os/setenv "JOLT_GITLIBS" "") +(os/cd "/") +(rmrf base) + +(if (> fails 0) + (error (string "deps-tasks-test: " fails " failing check(s)")) + (print "\nAll deps-tasks tests passed!"))