diff --git a/doc/tools-deps.md b/doc/tools-deps.md index 8f41ea7..7e8f4f7 100644 --- a/doc/tools-deps.md +++ b/doc/tools-deps.md @@ -73,8 +73,8 @@ hit interpreter gaps, which is fine). so the stdlib always wins. (Two small changes: a root list in the ctx, and trying `.cljc`.) 4. **Dev vs build.** - - *Dev:* on REPL/CLI start, if `deps.edn` is present, resolve once (cache keyed - on a hash of deps.edn so it's a no-op when unchanged) and register the roots. + - *Dev:* `jolt-deps` resolves `deps.edn` (cached on a hash of it, so it's a + no-op when unchanged) and runs jolt with the roots on `JOLT_PATH`; `(require '[medley.core])` then just works. - *Build:* the dep namespaces a project actually uses get compiled into the image the same way the embedded `jolt.nrepl` source is today — load them at @@ -87,14 +87,29 @@ semantics, or version conflict resolution. Resolution is a tree walk over git deps; "the classpath" is just the list of `src` dirs of the clones. The loader already does path-based namespace lookup — we widen it from one root to a few. -## Integration with jpm +## A separate tool, like jpm beside janet -jpm stays the build tool for the Jolt binary; this lives beside it and *calls -into* `jpm/pm` for the git/cache work (import the module at dev/build time — jpm -is always present then; the shipped binary doesn't need it). **Decision: reuse -`jpm/pm`** (`resolve-bundle` + `download-bundle`, after `jpm/config/load-default`) -rather than shelling `git` ourselves — less code, and it shares jpm's cache. The -internal-API risk is acceptable since it's only used at dev/build time. +The jolt *runtime* knows nothing about deps.edn — exactly as the `janet` binary +knows nothing about jpm. Resolution lives in a separate `jolt-deps` tool (its own +`declare-executable`), and the runtime's only interface is the source roots in +`JOLT_PATH`: + +``` +jolt-deps path # print the resolved roots, ':'-joined +jolt-deps run FILE [args] # resolve, then run `jolt FILE …` with JOLT_PATH set +jolt-deps repl # resolve, then start a jolt REPL with JOLT_PATH set +jolt-deps -e EXPR [args] # resolve, then `jolt -e …` +``` + +It **reuses `jpm/pm`** (`resolve-bundle` + `download-bundle`, after +`jpm/config/load-default`) for the git fetch/cache rather than shelling `git` — +less code, shares jpm's cache. jpm is loaded lazily (`require`, not `import`), so +it's pulled in only when resolving (dev time), never embedded in a binary. The +internal-API risk is acceptable since it's dev-time only. + +One gotcha worth recording: `jolt`'s context is built into its image at build +time, so `JOLT_PATH` is read at runtime in `main` (not in `init`, whose env read +would be frozen at build). ## Limitations @@ -111,12 +126,15 @@ internal-API risk is acceptable since it's only used at dev/build time. `init` adds roots from the `:paths` opt and `JOLT_PATH` (colon-separated). Verified loading a real lib (medley) from an added root. See `test/integration/deps-loader-test.janet`. -2. **Resolve git deps via jpm.** Read `deps.edn`, resolve `:git/*` (+ - `:local/root`) through `jpm/pm` into `jpm_tree/.cache`, recurse for transitive - git deps, register the roots. `jolt deps` to resolve/print; auto on startup - when `deps.edn` exists. +2. **Resolve git deps via jpm.** *(done)* `src/jolt/deps.janet` reads `deps.edn`, + resolves `:git/*` + `:local/root` through `jpm/pm` into `jpm_tree/.cache`, + recurses for transitive deps, and returns the roots (cached on a deps.edn + hash). The separate `jolt-deps` tool (`src/jolt/deps_cli.janet`, + `declare-executable`) exposes `path`/`run`/`repl`/`-e`. See + `test/integration/deps-resolve-test.janet`. 3. **Build-time compile-in.** Fold the used dep namespaces into the image at - build (as with embedded `jolt.nrepl`). + build (as with embedded `jolt.nrepl`), so a built artifact needs neither the + deps nor jpm. *(not started)* 4. **Conformance.** Pull a few popular pure-`cljc` git libs, see what loads/runs, and drive interpreter gaps from the failures — same loop as the clojure-test-suite battery. diff --git a/project.janet b/project.janet index 2676169..d6d9ddd 100644 --- a/project.janet +++ b/project.janet @@ -8,3 +8,9 @@ (declare-executable :name "jolt" :entry "src/jolt/main.janet") + +# Separate tool (like jpm beside janet): resolves deps.edn into Jolt source +# roots. The jolt runtime stays deps-agnostic — it just reads JOLT_PATH. +(declare-executable + :name "jolt-deps" + :entry "src/jolt/deps_cli.janet") diff --git a/src/jolt/deps.janet b/src/jolt/deps.janet new file mode 100644 index 0000000..2e487bf --- /dev/null +++ b/src/jolt/deps.janet @@ -0,0 +1,88 @@ +# deps.edn resolution for Jolt. +# +# Resolve git and :local/root dependencies from a deps.edn into a list of source +# directories, which the loader then searches (see evaluator/find-ns-file). We +# reuse jpm's git fetch + cache (jpm/pm) rather than shipping a package manager. +# Maven (:mvn/version) deps are ignored — git only, pure clj/cljc only. +# +# jpm is loaded lazily (require, not import) so it's needed only at resolve time +# (dev/build), never embedded in the shipped binary. + +# A typical deps.edn is also valid Janet data, so we read it with Janet's parser. +# (EDN-only forms — #{} sets, tagged literals, namespaced maps — aren't handled; +# deps.edn rarely uses them in :deps/:paths.) +(defn read-edn [path] + (when (os/stat path) + (try (parse (slurp path)) ([_] nil)))) + +(defn- jpm-fn [mod sym] + (get-in (require mod) [sym :value])) + +(defn- ensure-jpm-config [tree] + ((jpm-fn "jpm/config" 'load-default)) + (setdyn :modpath tree) + (setdyn :gitpath (dyn :gitpath "git"))) + +(defn- clone-git [spec] + # spec is a deps.edn dep value: {:git/url ... :git/sha/:git/tag ...} + (def resolve-bundle (jpm-fn "jpm/pm" 'resolve-bundle)) + (def download-bundle (jpm-fn "jpm/pm" 'download-bundle)) + (def b (resolve-bundle {:url (get spec :git/url) + :sha (get spec :git/sha) + :tag (get spec :git/tag) + :shallow false})) + (download-bundle (b :url) (b :type) (b :tag) (b :shallow))) + +(defn- src-roots + "Source dirs of a project/dep at `dir`: its deps.edn :paths joined to dir + (default [\"src\"])." + [dir edn] + (map |(string dir "/" $) (or (and edn (get edn :paths)) ["src"]))) + +(defn resolve-deps + "Resolve the git/:local deps of `deps-edn-path` into an ordered, de-duplicated + array of source dirs (the project's own :paths first, then each dependency's, + transitively). `tree` is where jpm's clone cache lives (default ./jpm_tree)." + [deps-edn-path &opt tree] + (default tree (string (os/cwd) "/jpm_tree")) + (os/mkdir tree) + (ensure-jpm-config tree) + (def roots @[]) + (def seen @{}) + (defn add-root [r] (unless (index-of r roots) (array/push roots r))) + (defn process [edn base-dir own-paths?] + (when (dictionary? edn) + (when own-paths? (each r (src-roots base-dir edn) (add-root r))) + (eachp [lib spec] (or (get edn :deps) {}) + (def k (string lib)) + (unless (get seen k) + (put seen k true) + (def dir + (cond + (and (dictionary? spec) (get spec :git/url)) (clone-git spec) + (and (dictionary? spec) (get spec :local/root)) + (let [lr (get spec :local/root)] + (if (string/has-prefix? "/" lr) lr (string base-dir "/" lr))) + nil)) # :mvn/* and anything else: skip + (when dir + (def dep-edn (read-edn (string dir "/deps.edn"))) + (each r (src-roots dir dep-edn) (add-root r)) + (process dep-edn dir false)))))) + (process (read-edn deps-edn-path) (os/cwd) true) + roots) + +(defn resolve-deps-cached + "Like resolve-deps, but caches the resolved roots in the tree keyed on a hash + of the deps.edn, so an unchanged deps.edn resolves without re-fetching." + [deps-edn-path &opt tree] + (default tree (string (os/cwd) "/jpm_tree")) + (when (os/stat deps-edn-path) + (os/mkdir tree) + (def cache-file (string tree "/.jolt-deps-roots.jdn")) + (def h (hash (slurp deps-edn-path))) + (def cached (when (os/stat cache-file) (try (parse (slurp cache-file)) ([_] nil)))) + (if (and cached (= h (get cached :hash))) + (get cached :roots) + (let [roots (resolve-deps deps-edn-path tree)] + (spit cache-file (string/format "%j" {:hash h :roots roots})) + roots)))) diff --git a/src/jolt/deps_cli.janet b/src/jolt/deps_cli.janet new file mode 100644 index 0000000..ae0c990 --- /dev/null +++ b/src/jolt/deps_cli.janet @@ -0,0 +1,41 @@ +# jolt-deps — a separate tool that resolves a deps.edn into Jolt source roots. +# +# Mirrors how jpm is a tool beside the janet runtime: the jolt runtime knows +# nothing about deps.edn — it just searches the roots in JOLT_PATH (see +# api/init). This tool does the resolution (git + :local deps, via jpm's fetch +# cache) and either prints the roots or launches jolt with JOLT_PATH set. +# +# jolt-deps path print the resolved roots (':'-joined), e.g. for +# JOLT_PATH=$(jolt-deps path) jolt file.clj +# jolt-deps run FILE [args] resolve, then run `jolt FILE args` with JOLT_PATH set +# jolt-deps repl resolve, then start a jolt REPL with JOLT_PATH set +# jolt-deps -e EXPR [args] resolve, then `jolt -e EXPR ...` with JOLT_PATH set +# +# The jolt binary is found via $JOLT_BIN, else `jolt` on PATH. + +(import ./deps :as deps) + +(defn- roots [] + (if (os/stat "deps.edn") (deps/resolve-deps-cached "deps.edn") @[])) + +(defn- exec-jolt [extra-args] + # Set JOLT_PATH in our own env and let the child inherit it (os/execute's env + # arg isn't honored here; inheriting is reliable). + (def rs (string/join (roots) ":")) + (def existing (os/getenv "JOLT_PATH")) + (os/setenv "JOLT_PATH" (if (and existing (> (length existing) 0)) (string rs ":" existing) rs)) + (os/execute [(os/getenv "JOLT_BIN" "jolt") ;extra-args] :p)) + +(defn- usage [] + (print "usage: jolt-deps [path | run FILE [args] | repl | -e EXPR [args]]")) + +(defn main [&] + (def argv (tuple/slice (or (dyn :args) @[]) 1)) + (def cmd (get argv 0)) + (cond + (or (nil? cmd) (= cmd "help") (= cmd "-h") (= cmd "--help")) (usage) + (= cmd "path") (print (string/join (roots) ":")) + (= cmd "run") (os/exit (exec-jolt (tuple/slice argv 1))) + (= cmd "repl") (os/exit (exec-jolt [])) + (= cmd "-e") (os/exit (exec-jolt argv)) + (do (eprint "jolt-deps: unknown command " cmd) (usage) (os/exit 1)))) diff --git a/src/jolt/main.janet b/src/jolt/main.janet index 06d9842..dca9fb2 100644 --- a/src/jolt/main.janet +++ b/src/jolt/main.janet @@ -309,6 +309,12 @@ (def args (or (dyn :args) @[])) # @["jolt" arg1 arg2 ...] (def argv (if (> (length args) 1) (array/slice args 1) @[])) (ctx-set-current-ns ctx "user") + # JOLT_PATH must be applied at runtime: this `ctx` is built into the image at + # build time, so its source-paths can't capture the runtime environment. + # `jolt-deps` sets JOLT_PATH to the resolved deps.edn source roots. + (when-let [jp (os/getenv "JOLT_PATH")] + (each p (string/split ":" jp) + (when (> (length p) 0) (array/push (get (ctx :env) :source-paths) p)))) (cond (empty? argv) (run-repl) (or (= (argv 0) "-h") (= (argv 0) "--help")) (print-help) diff --git a/test/integration/deps-resolve-test.janet b/test/integration/deps-resolve-test.janet new file mode 100644 index 0000000..f71c6fb --- /dev/null +++ b/test/integration/deps-resolve-test.janet @@ -0,0 +1,61 @@ +# deps.edn resolution into source roots, then loading a library through them. +# Uses :local/root deps only so it needs no network (the git path is the same +# code, exercised manually against real repos). + +(use ../../src/jolt/api) +(use ../../src/jolt/types) +(import ../../src/jolt/deps :as deps) + +(def base (string (os/getenv "TMPDIR") "/jolt-deps-resolve-" (os/time))) +(defn rmrf [p] + (when (os/stat p) + (if (= :directory (os/stat p :mode)) + (do (each e (os/dir p) (rmrf (string p "/" e))) (os/rmdir p)) + (os/rm p)))) +(rmrf base) + +(defn mkdirs [p] + (def abs (string/has-prefix? "/" p)) + (var acc nil) + (each seg (filter |(not= "" $) (string/split "/" p)) + (set acc (cond (nil? acc) (if abs (string "/" seg) seg) (string acc "/" seg))) + (unless (os/stat acc) (os/mkdir acc)))) + +# project -> depends on lib-a (local), which depends on lib-b (local, transitive) +(each d ["proj/src" "a/src/liba" "b/src/libb"] (mkdirs (string base "/" d))) +(spit (string base "/proj/deps.edn") `{:paths ["src"] :deps {my/a {:local/root "../a"}}}`) +(spit (string base "/a/deps.edn") `{:paths ["src"] :deps {my/b {:local/root "../b"}}}`) +(spit (string base "/b/deps.edn") `{:paths ["src"]}`) +(spit (string base "/a/src/liba/core.clj") "(ns liba.core (:require [libb.core :as b]))\n(defn val [] (b/n))\n") +(spit (string base "/b/src/libb/core.clj") "(ns libb.core)\n(defn n [] 99)\n") + +(var fails 0) +(defn check [label got expected] + (if (= got expected) (print " ok " label) + (do (++ fails) (printf " FAIL %s: want %q, got %q" label expected got)))) + +(os/cd (string base "/proj")) +(def roots (deps/resolve-deps "deps.edn" (string base "/proj/jpm_tree"))) +# roots: proj/src, a/src (transitive: b/src) — at least the two dep srcs present +(check "resolves transitive local dep roots" + (truthy? (and (some |(string/has-suffix? "/a/src" $) roots) + (some |(string/has-suffix? "/b/src" $) roots))) + true) + +# load through the resolved roots: liba.core requires libb.core transitively +(def ctx (init {:paths roots})) +(ctx-set-current-ns ctx "user") +(let [r (protect (eval-string ctx "(do (require (quote [liba.core :as a])) (a/val))"))] + (check "require local lib + transitive dep" (if (r 0) (r 1) (string "ERR " (r 1))) 99)) + +# cached resolution returns the same roots without re-walking +(check "cached resolve matches" + (deep= roots (deps/resolve-deps-cached "deps.edn" (string base "/proj/jpm_tree"))) + true) + +(os/cd "/") +(rmrf base) + +(if (> fails 0) + (error (string "deps-resolve-test: " fails " failing check(s)")) + (print "\nAll deps-resolve tests passed!"))