diff --git a/doc/include.md b/doc/include.md new file mode 100644 index 0000000..f8b115c --- /dev/null +++ b/doc/include.md @@ -0,0 +1,15 @@ +# Include Feature +## Requirements +The user can include page title, abstract or the whole content in a given page. Headings and enumerations can be indented by a given include-level. + +## Thoughts & Questions +* Which include syntax should be used? + * page include can be definde alongsite of image includes - sth. like + `&[:indent-heading s/Num :indent-list s/Num](relative or absolute url s/Str)` +* Which kind of urls should we accept for page includes? + * relative local urls (we will need some care to prohibit directory traversal ...) + * absolute github / gitlab / gitblit urls without authentication. +* Which metadata can be used for title / abstract ? + * MultiMarcdown-Metadata is supported by clj-markdown :-) +* How can we test page includes? + * we will need a content resolver component for testing and at least one for production resolving. diff --git a/project.clj b/project.clj index 38c96f5..9f20be0 100644 --- a/project.clj +++ b/project.clj @@ -29,7 +29,9 @@ [prone "1.1.4"] [ring/ring-anti-forgery "1.1.0"] [ring-server "0.4.0"] - [selmer "1.11.0"]] + [selmer "1.11.0"] + [com.stuartsierra/component "0.3.2"] + [prismatic/schema "1.1.9"]] :repl-options {:init-ns smeagol.repl} diff --git a/src/smeagol/include.clj b/src/smeagol/include.clj new file mode 100644 index 0000000..8f8a017 --- /dev/null +++ b/src/smeagol/include.clj @@ -0,0 +1,61 @@ +(ns ^{:doc "Functions related to the include of markdown-paged in a given markdown." + :author "Michael Jerger"} + smeagol.include + (:require + [clojure.string :as cs] + [schema.core :as s] + [com.stuartsierra.component :as component] + [smeagol.include.parse :as parse] + [smeagol.include.resolve :as resolve] + [smeagol.include.indent :as indent])) + +(s/defrecord Includer + [resolver]) + +(defprotocol IncludeMd + (expand-include-md + [includer md-src] + "return a markdown containing resolved includes")) + +(s/defn + do-expand-one-include :- s/Str + [includer :- Includer + include :- parse/IncludeLink + md-src :- s/Str] + (let [{:keys [uri replace indent-heading indent-list]} include] + (cs/replace-first + md-src + (re-pattern (cs/escape + replace + {\[ "\\[" + \] "\\]" + \( "\\(" + \) "\\)"})) + (indent/do-indent-list + indent-list + (indent/do-indent-heading + indent-heading + (resolve/resolve-md (:resolver includer) uri)))))) + +(s/defn + do-expand-includes :- s/Str + [includer :- Includer + includes :- [parse/IncludeLink] + md-src :- s/Str] + (loop [loop-includes includes + result md-src] + (if (empty? loop-includes) + result + (recur + (rest loop-includes) + (do-expand-one-include includer (first loop-includes) result))))) + +(extend-type Includer + IncludeMd + (expand-include-md [includer md-src] + (do-expand-includes includer (parse/parse-include-md md-src) md-src))) + +(s/defn + new-includer + [] + (map->Includer {})) diff --git a/src/smeagol/include/indent.clj b/src/smeagol/include/indent.clj new file mode 100644 index 0000000..f92a69c --- /dev/null +++ b/src/smeagol/include/indent.clj @@ -0,0 +1,58 @@ +(ns ^{:doc "Functions related to the include of markdown-paged - handling the +list & heading indents of includes. This namespaces is implementation detail for +smeagol.include and not inteded for direct usage." + :author "Michael Jerger"} + smeagol.include.indent + (:require + [clojure.string :as cs] + [schema.core :as s])) + +(s/defn + parse-list + [md-resolved :- s/Str] + (distinct + (into + (re-seq #"((^|\R? *)([\*\+-] ))" md-resolved) + (re-seq #"((^|\R? *)([0-9]+\. ))" md-resolved)))) + +(s/defn + parse-heading + [md-resolved :- s/Str] + (distinct + (re-seq #"((^|\R?)(#+ ))" md-resolved))) + +(s/defn + do-indent :- s/Str + [indent :- s/Num + indentor :- s/Str + elements + md-resolved :- s/Str] + (loop [result md-resolved + elements elements] + (if (empty? elements) + result + (let [element (first elements) + replace (nth element 1) + start (nth element 2) + end (nth element 3)] + (recur + (cs/replace + result + (re-pattern (cs/escape + replace + {\* "\\*" + \n "\\n"})) + (str start (apply str (repeat indent indentor)) end)) + (rest elements)))))) + +(s/defn + do-indent-heading :- s/Str + [indent :- s/Num + md-resolved :- s/Str] + (do-indent indent "#" (parse-heading md-resolved) md-resolved)) + +(s/defn + do-indent-list :- s/Str + [indent :- s/Num + md-resolved :- s/Str] + (do-indent indent " " (parse-list md-resolved) md-resolved)) diff --git a/src/smeagol/include/parse.clj b/src/smeagol/include/parse.clj new file mode 100644 index 0000000..0016252 --- /dev/null +++ b/src/smeagol/include/parse.clj @@ -0,0 +1,50 @@ +(ns ^{:doc "Functions related to the include of markdown-paged - parsing of +include links. This namespaces is implementation detail for +smeagol.include and not inteded for direct usage." + :author "Michael Jerger"} + smeagol.include.parse + (:require + [schema.core :as s])) + +(def IncludeLink + {:replace s/Str + :uri s/Str + :indent-heading s/Num + :indent-list s/Num}) + +(s/defn + convert-indent-to-int :- s/Num + [indents :- [s/Str]] + (if (some? indents) + (Integer/valueOf (nth indents 2)) + 0)) + +(s/defn + parse-indent-list + [md-src :- s/Str] + (re-matches #".*(:indent-list (\d)).*" md-src)) + +(s/defn + parse-indent-heading + [md-src :- s/Str] + (re-matches #".*(:indent-heading (\d)).*" md-src)) + +(s/defn + parse-include-link + [md-src :- s/Str] + (re-seq #".*(&\[\w*(.*)\w*\]\((.*)\)).*" md-src)) + +(s/defn + parse-include-md :- [IncludeLink] + [md-src :- s/Str] + (vec + (map + (fn [parse-element] + (let [replace (nth parse-element 1) + uri (nth parse-element 3) + indents-text (nth parse-element 2)] + {:replace replace + :uri uri + :indent-heading (convert-indent-to-int (parse-indent-heading indents-text)) + :indent-list (convert-indent-to-int (parse-indent-list indents-text))})) + (parse-include-link md-src)))) diff --git a/src/smeagol/include/resolve.clj b/src/smeagol/include/resolve.clj new file mode 100644 index 0000000..7a2b3b1 --- /dev/null +++ b/src/smeagol/include/resolve.clj @@ -0,0 +1,46 @@ +(ns ^{:doc "Functions related to the include of markdown-paged - providing +a plugable load-content componet. This namespaces is implementation detail for +smeagol.include and not inteded for direct usage." + :author "Michael Jerger"} + smeagol.include.resolve + (:require + [schema.core :as s] + [com.stuartsierra.component :as component])) + +(s/defrecord Resolver + [type :- s/Keyword + local-base-dir :- s/Str]) + +;As schema does'nt support s/defprotocol we use the dispatcher for annotation & validation. +(s/defn dispatch-by-resolver-type :- s/Keyword + "Dispatcher for different resolver implementations." + [resolver :- Resolver + uri :- s/Str] + (:type resolver)) + +(defmulti do-resolve-md + "Multimethod return a markfown file content for given uri." + dispatch-by-resolver-type) +(s/defmethod do-resolve-md :default + [resolver :- Resolver + uri :- s/Str] + (throw (Exception. (str "No implementation for " resolver)))) + +(defprotocol ResolveMd + (resolve-md + [resolver uri] + "return a markfown file content for given uru.")) + +(extend-type Resolver + ResolveMd + (resolve-md [resolver uri] + (s/validate s/Str uri) + (s/validate s/Str (do-resolve-md resolver uri)))) + +(s/defn + new-resolver + ([type :- s/Keyword] + (map->Resolver {:type type :local-base-dir nil})) + ([type :- s/Keyword + local-base-dir :- s/Str] + (map->Resolver {:type type :local-base-dir local-base-dir}))) diff --git a/src/smeagol/include/resolve_local_file.clj b/src/smeagol/include/resolve_local_file.clj new file mode 100644 index 0000000..c35e3e5 --- /dev/null +++ b/src/smeagol/include/resolve_local_file.clj @@ -0,0 +1,27 @@ +(ns ^{:doc "Functions related to the include of markdown-paged - providing +a plugable load-local-include-links componet. This namespaces is implementation detail for +smeagol.include and not inteded for direct usage." + :author "Michael Jerger"} + smeagol.include.resolve-local-file + (:require + [schema.core :as s] + [smeagol.include.resolve :as resolve] + [com.stuartsierra.component :as component] + [clojure.java.io :as cjio] + [taoensso.timbre :as timbre])) + +(s/defmethod resolve/do-resolve-md :local-file + [resolver + uri :- s/Str] + (let [file-name uri + file-path (cjio/file (:local-base-dir resolver) file-name) + exists? (.exists (clojure.java.io/as-file file-path))] + (cond exists? + (do + (timbre/info (format "Including page '%s' from file '%s'" uri file-path)) + (slurp file-path))))) + +(s/defn + new-resolver + [local-base-dir :- s/Str] + (resolve/new-resolver :local-file local-base-dir)) diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index fe80349..109a2cc 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -20,7 +20,10 @@ [smeagol.sanity :refer [show-sanity-check-error]] [smeagol.util :as util] [smeagol.uploads :as ul] - [taoensso.timbre :as timbre])) + [taoensso.timbre :as timbre] + [com.stuartsierra.component :as component] + [smeagol.include.resolve-local-file :as resolve] + [smeagol.include :as include])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; @@ -108,6 +111,14 @@ (edit-page request "stylesheet" ".css" "edit-css.html" "_edit-side-bar.md")) +(def md-include-system + (component/start + (component/system-map + :resolver (resolve/new-resolver util/content-dir) + :includer (component/using + (include/new-includer) + [:resolver])))) + (defn wiki-page "Render the markdown page specified in this `request`, if any. If none found, redirect to edit-page" [request] @@ -125,7 +136,10 @@ (merge (util/standard-params request) {:title page :page page - :content (md->html (slurp file-path)) + :content (md->html + (include/expand-include-md + (:includer md-include-system) + (slurp file-path))) :editable true}))) true (response/redirect (str "/edit?page=" page)))))) diff --git a/test/smeagol/test/include.clj b/test/smeagol/test/include.clj new file mode 100644 index 0000000..3a037f1 --- /dev/null +++ b/test/smeagol/test/include.clj @@ -0,0 +1,106 @@ +(ns smeagol.test.include + (:require [clojure.test :refer :all] + [schema.core :as s] + [com.stuartsierra.component :as component] + [smeagol.include.resolve :as resolve] + [smeagol.include :as sut])) + +(def include-simple + "# Heading1 +&[](./simple.md)") + +(def include-surounding-simple + "# Heading1 +Some surounding &[](./simple.md) text") + +(def include-heading-0 + "# Heading1 +&[:indent-heading 0](./with-heading.md)") + +(def include-heading-list-1 + "# Heading1 +&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)") + +(def include-heading-list-0 + "# Heading1 +&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)") + +(def include-invalid-indent + "# Heading1 +&[ invalid input should default to indent 0 ](./simple.md)") + +(def include-spaced-indent + "# Heading1 +&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md)") + +(def multi + "# Heading1 +&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md) +some text +&[](./simple.md) +more text.") + +(s/defmethod resolve/do-resolve-md :test-mock + [resolver + uri :- s/Str] + (cond + (= uri "./simple.md") "Simple content." + (= uri "./with-heading-and-list.md") "# Heading2 +some text +* List + +## Heading 3 +more text")) + + + +(def system-under-test + (component/start + (component/system-map + :resolver (resolve/new-resolver :test-mock) + :includer (component/using + (sut/new-includer) + [:resolver])))) + +(deftest test-expand-include-md + (testing "The whole integration of include" + (is + (= "# Heading" + (sut/expand-include-md (:includer system-under-test) "# Heading"))) + (is + (= "# Heading1 +Simple content." + (sut/expand-include-md + (:includer system-under-test) + include-simple))) + (is + (= "# Heading1 +Some surounding Simple content. text" + (sut/expand-include-md + (:includer system-under-test) + include-surounding-simple))) + (is + (= "# Heading1 +# Heading2 +some text +* List + +## Heading 3 +more text" + (sut/expand-include-md + (:includer system-under-test) + include-heading-list-0))) + (is + (= "# Heading1 +### Heading2 +some text + * List + +#### Heading 3 +more text +some text +Simple content. +more text." + (sut/expand-include-md + (:includer system-under-test) + multi))))) diff --git a/test/smeagol/test/include/indent.clj b/test/smeagol/test/include/indent.clj new file mode 100644 index 0000000..b4ca363 --- /dev/null +++ b/test/smeagol/test/include/indent.clj @@ -0,0 +1,35 @@ +(ns smeagol.test.include.indent + (:require [clojure.test :refer :all] + [smeagol.include.indent :as sut])) + +(deftest test-parse-heading + (testing + (is (= '(["# " "# " "" "# "]) + (sut/parse-heading "# h1"))) + (is (= '(["\n# " "\n# " "\n" "# "]) + (sut/parse-heading "\n# h1"))))) + +(deftest test-indent-heading + (testing + (is (= "# h1" + (sut/do-indent-heading 0 "# h1"))) + (is (= "### h1" + (sut/do-indent-heading 2 "# h1"))) + (is (= "\n### h1" + (sut/do-indent-heading 2 "\n# h1"))))) + +(deftest test-parse-list + (testing + (is (= '([" * " " * " " " "* "]) + (sut/parse-list " * list"))) + (is (= '(["\n * " "\n * " "\n " "* "]) + (sut/parse-list "\n * list"))))) + +(deftest test-indent-list + (testing + (is (= " * list" + (sut/do-indent-list 0 " * list"))) + (is (= " * list" + (sut/do-indent-list 2 " * list"))) + (is (= "\n * list" + (sut/do-indent-list 2 "\n * list"))))) diff --git a/test/smeagol/test/include/parse.clj b/test/smeagol/test/include/parse.clj new file mode 100644 index 0000000..af27abf --- /dev/null +++ b/test/smeagol/test/include/parse.clj @@ -0,0 +1,91 @@ +(ns smeagol.test.include.parse + (:require [clojure.test :refer :all] + [schema.core :as s] + [smeagol.include.parse :as sut])) + +(def include-simple + "# Heading1 +&[](./simple.md)") + +(def include-surounding-simple + "# Heading1 +Some surounding &[](./simple.md) text") + +(def include-heading-0 + "# Heading1 +&[:indent-heading 0](./with-heading.md)") + +(def include-heading-list-1 + "# Heading1 +&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)") + +(def include-heading-list-0 + "# Heading1 +&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)") + +(def include-invalid-indent + "# Heading1 +&[ invalid input should default to indent 0 ](./simple.md)") + +(def include-spaced-indent + "# Heading1 +&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md)") + +(def multi + "# Heading1 +&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md) +some text +&[](./simple.md) +more text.") + + +(deftest test-parse-include-md + (testing "parse include links" + (is + (= [] + (sut/parse-include-md "# Heading"))) + (is + (= [{:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + include-simple))) + (is + (= [{:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + include-surounding-simple))) + (is + (= [{:replace "&[:indent-heading 0](./with-heading.md)" :uri "./with-heading.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + include-heading-0))) + (is + (= [{:replace + "&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)" + :uri "./with-heading-and-list.md", :indent-heading 1, :indent-list 1}] + (sut/parse-include-md + include-heading-list-1))) + (is + (= [{:replace + "&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)" + :uri "./with-heading-and-list.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + include-heading-list-0))) + (is + (= [{:replace + "&[ invalid input should default to indent 0 ](./simple.md)" + :uri "./simple.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + include-invalid-indent))) + (is + (= [{:replace + "&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md)" + :uri "./with-heading-and-list.md", :indent-heading 2, :indent-list 3}] + (sut/parse-include-md + include-spaced-indent))) + (is + (= [{:replace + "&[ :indent-heading 2 :indent-list 33 ](./with-heading-and-list.md)" + :uri "./with-heading-and-list.md", + :indent-heading 2, + :indent-list 3} + {:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}] + (sut/parse-include-md + multi))))) diff --git a/test/smeagol/test/include/resolve.clj b/test/smeagol/test/include/resolve.clj new file mode 100644 index 0000000..4da32ed --- /dev/null +++ b/test/smeagol/test/include/resolve.clj @@ -0,0 +1,8 @@ +(ns smeagol.test.include.resolve + (:require [clojure.test :refer :all] + [smeagol.include.resolve :as sut])) + +(deftest test-local-links + (testing "Rewriting of local links" + (is (thrown? Exception + (sut/resolve-md (sut/new-resolver (:default)) "./some-uri.md")))))