Add a Clojure FFI so libraries can bind native code (jolt.ffi)

A jolt library can now bind its own native dependencies and expose a Clojure API
over them — no jolt built-in required. This is the foundation for moving the
http-client / db / adapter functionality out of the host and into real libraries.

- jolt.ffi/foreign-fn (sugar: defcfn) is a compiler special form: a compile-time
  -typed C signature lowers to a real Chez foreign-procedure (analyzer :ffi-fn ->
  backend foreign-procedure), so calls are typed and marshaled, not eval'd.
- host/chez/ffi.ss provides the rest under jolt.ffi: load-library, alloc/free,
  read/write/sizeof, ptr<->string, null/null?. Loaded after the loader snapshot
  so a library's (require '[jolt.ffi]) still loads the macro side.
- Types: int/uint/long/ulong/int64/uint64/size_t/ssize_t/iptr/uptr/double/float/
  pointer/string/void/uint8/char.

Validated end to end: a pure-Clojure file binds libc (getpid/strlen/abs) and
libsqlite3 (open/prepare/step/column/finalize over out-param pointers) and runs a
query. Gate test test/chez/ffi-binding-test.ss (make ffi); selfhost holds.
This commit is contained in:
Yogthos 2026-06-22 10:59:51 -04:00
parent ccf93c896a
commit 537cb360b4
8 changed files with 282 additions and 62 deletions

View file

@ -22,8 +22,12 @@
(load "host/chez/png.ss") ; jolt.png — a baked namespace before the snapshot
(load "host/chez/http-client.ss") ; jolt.http-client (libcurl)
(load "host/chez/sqlite.ss") ; jolt.sqlite + jdbc.core (libsqlite3)
(load "host/chez/http-server.ss") ; jolt.http.server + ring-janet.adapter (BSD sockets)
(load "host/chez/http-server.ss") ; jolt.http.server (BSD sockets)
(load "host/chez/loader.ss")
;; jolt.ffi host primitives (memory / library loading) load AFTER the loader's
;; baked-ns snapshot, so a library's (require '[jolt.ffi]) still loads jolt.ffi's
;; Clojure side (the foreign-fn / defcfn macros, src/jolt/jolt/ffi.clj).
(load "host/chez/ffi.ss") ; jolt.ffi (FFI: a library binds native code)
;; jolt.main + jolt.deps live under jolt-core; keep them (and src/jolt) on the
;; roots so the CLI's own namespaces — and any jolt.* an app pulls in — resolve.

98
host/chez/ffi.ss Normal file
View file

@ -0,0 +1,98 @@
;; ffi.ss — the runtime side of jolt's foreign-function interface (jolt.ffi).
;;
;; A jolt LIBRARY binds native code itself: it loads a shared object and declares
;; typed foreign functions, then exposes a Clojure API. The TYPED CALL is lowered
;; at compile time to a Chez `foreign-procedure` by the backend (the
;; `jolt.ffi/foreign-fn` special form) — this file provides everything that does
;; NOT need compile-time types: loading libraries, allocating/reading/writing
;; foreign memory, and string/pointer marshaling. All exposed under `jolt.ffi`.
;;
;; A foreign pointer is a Chez machine address (an exact integer / uptr), the same
;; representation `void*` arguments and results use, so pointers flow between
;; foreign-fn calls and these helpers transparently.
;; --- loading shared objects --------------------------------------------------
;; (jolt.ffi/load-library name) loads a .so/.dylib by name (resolved by the OS
;; loader against the standard search paths). A library typically calls this once
;; at load with a platform-specific name. (load-library) with no name (or #f)
;; loads the running process's own symbols (libc, sockets).
(define (ffi-load-library . args)
(if (or (null? args) (jolt-nil? (car args)))
(begin (load-shared-object #f) jolt-nil)
(begin (load-shared-object (jolt-str-render-one (car args))) jolt-nil)))
(define (ffi-loaded? name)
(guard (e (#t #f)) (load-shared-object (jolt-str-render-one name)) #t))
;; --- foreign type keywords ---------------------------------------------------
;; The keyword type names jolt.ffi accepts (in foreign-fn signatures and the
;; memory accessors) map to Chez foreign types. Kept in one place so the backend
;; (compile-time, for foreign-procedure) and these accessors (runtime, for
;; foreign-ref/set!) agree.
(define (ffi-type->chez kw)
(let ((n (if (keyword-t? kw) (keyword-t-name kw) (jolt-str-render-one kw))))
(cond
((string=? n "int") 'int)
((string=? n "uint") 'unsigned-int)
((string=? n "long") 'long)
((string=? n "ulong") 'unsigned-long)
((string=? n "int64") 'integer-64)
((string=? n "uint64") 'unsigned-64)
((string=? n "size_t") 'size_t)
((string=? n "ssize_t") 'ssize_t)
((string=? n "iptr") 'iptr)
((string=? n "uptr") 'uptr)
((string=? n "double") 'double)
((string=? n "float") 'float)
((or (string=? n "pointer") (string=? n "void*")) 'void*)
((string=? n "string") 'string)
((string=? n "void") 'void)
((or (string=? n "uint8") (string=? n "u8") (string=? n "byte")) 'unsigned-8)
((string=? n "char") 'char)
(else (error #f (string-append "jolt.ffi: unknown foreign type :" n))))))
;; --- foreign memory ----------------------------------------------------------
;; alloc returns a pointer (integer address). The caller frees it. read/write take
;; a type keyword and an optional byte offset.
(define (ffi-alloc nbytes) (foreign-alloc (jnum->exact nbytes)))
(define (ffi-free ptr) (foreign-free (jnum->exact ptr)) jolt-nil)
(define (ffi-read ptr ty . off)
(foreign-ref (ffi-type->chez ty) (jnum->exact ptr) (if (pair? off) (jnum->exact (car off)) 0)))
(define (ffi-write ptr ty off val)
(foreign-set! (ffi-type->chez ty) (jnum->exact ptr) (jnum->exact off) val) jolt-nil)
;; sizeof a foreign type (for laying out structs / arrays).
(define (ffi-sizeof ty) (foreign-sizeof (ffi-type->chez ty)))
(define (ffi-null? ptr) (and (number? ptr) (= (jnum->exact ptr) 0)))
(define ffi-null 0)
;; --- string / bytevector marshaling ------------------------------------------
;; A C string result already comes back as a jolt string (the `string` foreign
;; type). For a `void*` that points at a NUL-terminated C string, read it here.
(define (ffi-ptr->string ptr)
(if (ffi-null? ptr) jolt-nil
(let ((p (jnum->exact ptr)))
(let loop ((i 0) (acc '()))
(let ((b (foreign-ref 'unsigned-8 p i)))
(if (= b 0) (utf8->string (u8-list->bytevector (reverse acc)))
(loop (+ i 1) (cons b acc))))))))
;; Copy a jolt string's UTF-8 bytes into a freshly alloc'd NUL-terminated buffer;
;; the caller frees it. Returns the pointer.
(define (ffi-string->ptr s)
(let* ((bv (string->utf8 (jolt-str-render-one s))) (n (bytevector-length bv))
(p (foreign-alloc (+ n 1))))
(do ((i 0 (+ i 1))) ((= i n)) (foreign-set! 'unsigned-8 p i (bytevector-u8-ref bv i)))
(foreign-set! 'unsigned-8 p n 0)
p))
;; --- expose under jolt.ffi ---------------------------------------------------
(def-var! "jolt.ffi" "load-library" ffi-load-library)
(def-var! "jolt.ffi" "loaded?" (lambda (n) (if (ffi-loaded? n) #t #f)))
(def-var! "jolt.ffi" "alloc" ffi-alloc)
(def-var! "jolt.ffi" "free" ffi-free)
(def-var! "jolt.ffi" "read" ffi-read)
(def-var! "jolt.ffi" "write" ffi-write)
(def-var! "jolt.ffi" "sizeof" ffi-sizeof)
(def-var! "jolt.ffi" "null?" (lambda (p) (if (ffi-null? p) #t #f)))
(def-var! "jolt.ffi" "null" ffi-null)
(def-var! "jolt.ffi" "ptr->string" ffi-ptr->string)
(def-var! "jolt.ffi" "string->ptr" ffi-string->ptr)

File diff suppressed because one or more lines are too long