Merge pull request #5 from jolt-lang/deps-phase34

deps phases 3-4: uberscript bundling + conformance harness

doc/building-and-deps.md

@@ -93,5 +93,17 @@ so an unchanged `deps.edn` doesn't re-fetch.
   or fail at a call. Coverage is per-function: a namespace can load with most
   functions working and a few not.
 
-See [`tools-deps.md`](tools-deps.md) for the design rationale and the road to
+### Bundling into one file
-compiling used dependency namespaces into the build.
+
+`jolt uberscript OUT.clj -m NS` (or `jolt-deps uberscript …`, which resolves deps
+first) bundles `NS` and every namespace it requires — your code plus its
+dependencies — into a single `.clj` in dependency order, ending with a call to
+`NS/-main`. The result runs on a plain `jolt` with no `JOLT_PATH`, no deps
+fetched, and no jpm:
+
+```bash
+jolt-deps uberscript app.clj -m myapp.main
+jolt app.clj arg1 arg2
+```
+
+See [`tools-deps.md`](tools-deps.md) for the design rationale.

doc/tools-deps.md

@@ -74,9 +74,22 @@ env read would be frozen at build).
   `JOLT_PATH`/`:paths`. Test: `test/integration/deps-loader-test.janet`.
 - **Resolve git/local deps via jpm** — done. `deps.janet` + the `jolt-deps` tool.
   Test: `test/integration/deps-resolve-test.janet`.
-- **Build-time compile-in** — not started. Fold the dep namespaces a project uses
+- **Bundling (uberscript)** — done. `jolt uberscript OUT -m NS` requires `NS`,
-  into the image at build (as with the embedded `jolt.nrepl` source), so a built
+  records the load order the loader emits (a dependency finishes loading before
-  artifact needs neither the deps nor jpm.
+  the file that required it, so it's topological), and concatenates the source
-- **Conformance** — not started. Pull popular pure-`cljc` git libs, see what
+  into one `.clj` that runs on a plain `jolt` — no deps, no jpm. `jolt-deps
-  loads/runs, and drive interpreter gaps from the failures — the same loop as the
+  uberscript` resolves first. Test: `test/integration/uberscript-test.janet`.
-  clojure-test-suite battery.
+  (This is the practical form of "compile the used namespaces into the build";
+  embedding into a custom binary image is a heavier variant we haven't needed.)
+- **Conformance** — harness in place:
+  `test/integration/deps-conformance-test.janet` resolves real pure-`cljc` git
+  libs and reports load/run status (network-gated behind `JOLT_CONFORMANCE=1`, so
+  CI stays offline). First run:
+  - `medley` — loads and runs.
+  - `cuerdas` — fails to load: the reader rejects a char/literal it uses
+    (`Unsupported character: \`). A genuine reader gap to chase.
+  - `stuartsierra/dependency` — `Long/MAX_VALUE`: JVM interop, so out of scope
+    (it isn't actually pure-`cljc`).
+
+  The loop from here is the same as the clojure-test-suite battery: add libs,
+  see what breaks, fix interpreter/reader gaps.

src/jolt/deps_cli.janet

@@ -10,6 +10,8 @@
 #   jolt-deps run FILE [args] resolve, then run `jolt FILE args` with JOLT_PATH set
 #   jolt-deps repl            resolve, then start a jolt REPL with JOLT_PATH set
 #   jolt-deps -e EXPR [args]  resolve, then `jolt -e EXPR ...` with JOLT_PATH set
+#   jolt-deps uberscript OUT -m NS
+#                             resolve, then bundle NS + its deps into one .clj
 #
 # The jolt binary is found via $JOLT_BIN, else `jolt` on PATH.
 
@@ -38,4 +40,5 @@
     (= cmd "run")  (os/exit (exec-jolt (tuple/slice argv 1)))
     (= cmd "repl") (os/exit (exec-jolt []))
     (= cmd "-e")   (os/exit (exec-jolt argv))
+    (= cmd "uberscript") (os/exit (exec-jolt argv))
     (do (eprint "jolt-deps: unknown command " cmd) (usage) (os/exit 1))))

src/jolt/evaluator.janet

@@ -228,6 +228,9 @@
             # (their defs intern there); a library's own `ns` form overrides this.
             (ctx-set-current-ns ctx ns-name)
             (load-ns-file ctx path)
+            # Record load order for tooling (uberscript): a dependency finishes
+            # loading before the file that required it, so this is topological.
+            (when-let [lf (get (ctx :env) :loaded-files)] (array/push lf path))
             (ctx-set-current-ns ctx saved)))))))
 
 (defn- eval-require

src/jolt/main.janet

@@ -324,6 +324,30 @@
       (load-string ctx (string "(apply " ns-name "/-main *command-line-args*)")))
     ([err fib] (report-error err fib) (os/exit 1))))
 
+(defn- run-uberscript [out main-ns]
+  # Bundle main-ns and everything it requires (from JOLT_PATH roots) into one
+  # .clj that runs on a plain jolt — no deps, no jpm. We require the entry and
+  # collect the load order the loader records (deps before dependents).
+  (when (or (nil? out) (nil? main-ns))
+    (eprint "Usage: jolt uberscript OUT.clj -m NS") (os/exit 1))
+  (put (ctx :env) :loaded-files @[])
+  (try
+    (load-string ctx (string "(require '[" main-ns "])"))
+    ([err fib] (report-error err fib) (os/exit 1)))
+  (def seen @{})
+  (def files @[])
+  (each f (get (ctx :env) :loaded-files)
+    (unless (get seen f) (put seen f true) (array/push files f)))
+  (def buf @"")
+  (buffer/push-string buf (string ";; Generated by `jolt uberscript` — " (length files) " namespace(s)\n\n"))
+  (each f files
+    (buffer/push-string buf (string ";; --- " f " ---\n"))
+    (buffer/push-string buf (slurp f))
+    (buffer/push-string buf "\n"))
+  (buffer/push-string buf (string "\n(apply " main-ns "/-main *command-line-args*)\n"))
+  (spit out (string buf))
+  (print "Wrote " out " (" (length files) " namespace(s))"))
+
 (defn- print-help []
   (print "Jolt — a Clojure interpreter on Janet\n")
   (print "Usage: jolt [opt] [args]\n")
@@ -335,6 +359,7 @@
   (print "  -m, --main NS [args]  Require NS and call its -main with the remaining args")
   (print "  nrepl-server [addr]   Start an nREPL server (addr = [host:]port, default 7888)")
   (print "                          (aliases: --nrepl-server, nrepl)")
+  (print "  uberscript OUT -m NS  Bundle NS + its required namespaces into one .clj")
   (print "  --version, version    Print the Jolt version")
   (print "  -h, --help, help      Show this help\n")
   (print "Dependencies (deps.edn) are handled by the separate jolt-deps tool."))
@@ -365,5 +390,10 @@
     (eval-flags (argv 0)) (run-eval (get argv 1 "") (array/slice argv 2))
     (file-flags (argv 0)) (run-file (get argv 1) (array/slice argv 2))
     (main-flags (argv 0)) (run-main (get argv 1) (array/slice argv 2))
+    (= (argv 0) "uberscript")
+      (let [out (get argv 1)
+            rest (array/slice argv 2)
+            mi (or (index-of "-m" rest) (index-of "--main" rest))]
+        (run-uberscript out (if mi (get rest (+ mi 1)) nil)))
     (= (argv 0) "-") (run-file "/dev/stdin" (array/slice argv 1))
     (run-file (argv 0) (array/slice argv 1))))

test/integration/deps-conformance-test.janet

@@ -0,0 +1,54 @@
+# Conformance pass for deps.edn-loaded libraries: resolve a few real, pure-cljc
+# git libraries and check that their namespaces load and a sample call works.
+#
+# Network-gated: set JOLT_CONFORMANCE=1 to run (it clones from GitHub). Skipped by
+# default so CI stays offline. Findings are summarized in doc/tools-deps.md.
+
+(use ../../src/jolt/api)
+(use ../../src/jolt/types)
+(import ../../src/jolt/deps :as deps)
+
+(unless (os/getenv "JOLT_CONFORMANCE")
+  (print "deps-conformance: set JOLT_CONFORMANCE=1 to run (needs network) — skipped")
+  (os/exit 0))
+
+(def libs
+  [{:name "medley" :url "https://github.com/weavejester/medley" :tag "1.4.0"
+    :ns "medley.core" :check "(medley.core/find-first odd? [2 4 5])" :expect "5"}
+   {:name "cuerdas" :url "https://github.com/funcool/cuerdas" :tag "2022.06.16-403"
+    :ns "cuerdas.core" :check "(cuerdas.core/kebab \"helloWorld\")" :expect "hello-world"}
+   {:name "dependency" :url "https://github.com/stuartsierra/dependency" :tag "dependency-1.0.0"
+    :ns "com.stuartsierra.dependency"
+    :check "(boolean (com.stuartsierra.dependency/graph))" :expect "true"}])
+
+(def base (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-conf-" (os/time)))
+(os/mkdir base)
+
+(defn- try-lib [lib]
+  (def dir (string base "/" (lib :name)))
+  (os/mkdir dir)
+  (spit (string dir "/deps.edn")
+        (string "{:deps {the/lib {:git/url \"" (lib :url) "\" :git/tag \"" (lib :tag) "\"}}}"))
+  (def prev (os/cwd))
+  (defer (os/cd prev)
+    (os/cd dir)
+    (def r (protect (deps/resolve-deps "deps.edn")))
+    (if (not (r 0))
+      [:resolve-failed (string (r 1))]
+      (let [ctx (init {:paths (r 1)})]
+        (ctx-set-current-ns ctx "user")
+        (def lr (protect (eval-string ctx (string "(require (quote [" (lib :ns) "]))"))))
+        (if (not (lr 0))
+          [:load-failed (string (lr 1))]
+          (let [cr (protect (eval-string ctx (lib :check)))]
+            (cond
+              (not (cr 0)) [:check-error (string (cr 1))]
+              (= (string (normalize-pvecs (cr 1))) (lib :expect)) [:ok nil]
+              [:check-mismatch (string "got " (string (normalize-pvecs (cr 1))))])))))))
+
+(print "deps conformance — pure-cljc git libs:\n")
+(each lib libs
+  (def [status detail] (try (try-lib lib) ([err] [:crash (string err)])))
+  (printf "  %-12s %-16s %s" (lib :name) status (or detail "")))
+(print "")
+(os/exit 0)

test/integration/uberscript-test.janet

@@ -0,0 +1,51 @@
+# `jolt uberscript` bundles a namespace and everything it requires into one .clj
+# that runs on a plain jolt with no JOLT_PATH / deps. Runs from source.
+
+(defn- run [env-jolt-path & args]
+  (if env-jolt-path (os/setenv "JOLT_PATH" env-jolt-path) (os/setenv "JOLT_PATH" nil))
+  (def p (os/spawn ["janet" "src/jolt/main.janet" ;args] :p {:out :pipe :err :pipe}))
+  (def out (:read (p :out) :all))
+  (os/proc-wait p)
+  (string (or out "")))
+
+(defn- mkdirs [p]
+  (var acc nil)
+  (each seg (filter |(not= "" $) (string/split "/" p))
+    (set acc (if (nil? acc) (if (string/has-prefix? "/" p) (string "/" seg) seg) (string acc "/" seg)))
+    (unless (os/stat acc) (os/mkdir acc))))
+(defn- rmrf [p]
+  (when (os/stat p)
+    (if (= :directory (os/stat p :mode))
+      (do (each e (os/dir p) (rmrf (string p "/" e))) (os/rmdir p))
+      (os/rm p))))
+
+(def base (string (or (os/getenv "TMPDIR") "/tmp") "/jolt-uber-" (os/time)))
+(rmrf base)
+(mkdirs (string base "/proj/src/app"))
+(mkdirs (string base "/lib/src/greet"))
+(spit (string base "/lib/src/greet/core.clj")
+      "(ns greet.core)\n(defn hello [n] (str \"Hello, \" n \"!\"))\n")
+(spit (string base "/proj/src/app/core.clj")
+      "(ns app.core (:require [greet.core :as g]))\n(defn -main [& args] (println (g/hello (or (first args) \"world\"))))\n")
+
+(var fails 0)
+(defn check [label got pred]
+  (if (pred got) (print "  ok   " label)
+    (do (++ fails) (printf "  FAIL %s: got %q" label got))))
+(defn- has [s] (fn [x] (string/find s x)))
+
+(def roots (string base "/proj/src:" base "/lib/src"))
+(def out (string base "/out.clj"))
+
+# build the uberscript with the dep roots on JOLT_PATH
+(run roots "uberscript" out "-m" "app.core")
+(check "uberscript written" (if (os/stat out) "yes" "no") (has "yes"))
+(check "bundles the dep ns" (slurp out) (has "(ns greet.core)"))
+
+# run it standalone: no JOLT_PATH, so it only works if the dep was inlined
+(check "runs standalone" (run nil out "Bob") (has "Hello, Bob!"))
+
+(rmrf base)
+(if (> fails 0)
+  (error (string "uberscript-test: " fails " failing check(s)"))
+  (print "\nAll uberscript tests passed!"))