Merge remote-tracking branch 'origin/develop' into develop

README.md

@@ -98,6 +98,18 @@ You can (if you're logged in) upload files, including images, using the **Upload
 
 ![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509)
 
+## Includes
+You can include pages into the current page you're working on. To do so, you can add a include link:
+
+`&[:indent-heading s/Num :indent-list s/Num](relative or absolute uri s/Str)`
+
+Parameters semantics:
+ * uri: The page to include. At the moment only pages from the current wiki are allowed. A page called "PageToBeIncluded" will result in a uri "PageToBeIncluded.md".
+ * indent-heading: You can indent headings of included page to adjust the included content to your surrounding structure. Indents 0-9 are supported.
+ * indent-list: In Same manner you can indent lists of included page to adjust the included content to your surrounding structure. Indents 0-9 are supported.
+
+ Security warning: At the moment there is no check against directory traversal attack. So include feature may expose files outside of your wiki content-dir.
+
 ## Advertisement
 If you like what you see here, I am available for work on open source Clojure projects.
 

doc/include.md

@@ -0,0 +1,15 @@
+# Include Feature
+## Requirements
+The user can include page title, abstract or the whole content in a given page. Headings and enumerations can be indented by a given include-level.
+
+## Thoughts & Questions
+* Which include syntax should be used?
+  * page include can be definde alongsite of image includes - sth. like  
+    `&[:indent-heading s/Num :indent-list s/Num](relative or absolute url s/Str)`
+* Which kind of urls should we accept for page includes?
+  * relative local urls (we will need some care to prohibit directory traversal ...)
+  * absolute github / gitlab / gitblit urls without authentication.
+* Which metadata can be used for title / abstract ?
+  * MultiMarcdown-Metadata is supported by clj-markdown :-)
+* How can we test page includes?
+  * we will need a content resolver component for testing and at least one for production resolving.

project.clj

@@ -1,4 +1,4 @@
-(defproject smeagol "1.0.1"
+(defproject smeagol "1.0.2-SNAPSHOT"
   :description "A simple Git-backed Wiki inspired by Gollum"
   :url "https://github.com/simon-brooke/smeagol"
   :license {:name "GNU General Public License,version 2.0 or (at your option) any later version"
@@ -7,6 +7,7 @@
                  [clj-yaml "0.4.0"]
                  [com.cemerick/url "0.1.1"]
                  [com.fzakaria/slf4j-timbre "0.3.7"]
+                 [com.stuartsierra/component "0.3.2"]
                  [com.taoensso/encore "2.92.0"]
                  [com.taoensso/timbre "4.10.0"]
                  [com.taoensso/tower "3.0.2" :exclusions [com.taoensso/encore]]
@@ -26,6 +27,7 @@
                  [org.slf4j/log4j-over-slf4j "1.7.25"]
                  [org.slf4j/jul-to-slf4j "1.7.25"]
                  [org.slf4j/jcl-over-slf4j "1.7.25"]
+                 [prismatic/schema "1.1.9"]
                  [prone "1.1.4"]
                  [ring/ring-anti-forgery "1.1.0"]
                  [ring-server "0.4.0"]

resources/config.edn

@@ -28,16 +28,16 @@
 ;;      ;       ;       ;       ;       ;       ;       ;       ;       ;
 {
   :content-dir          "resources/public/content"
+  :start-page           "Introduction"
                                         ;; where content is served from.
-  :default-locale       "lt-LT"         ;; default language used for messages
+  :default-locale       "en-GB"         ;; default language used for messages
   :formatters           {"vega"         smeagol.formatting/process-vega
                          "vis"          smeagol.formatting/process-vega
                          "mermaid"      smeagol.formatting/process-mermaid
                          "backticks"    smeagol.formatting/process-backticks}
-  :log-level            :trace           ;; the minimum logging level; one of
+  :log-level            :info           ;; the minimum logging level; one of
                                         ;; :trace :debug :info :warn :error :fatal
   :passwd               "resources/passwd"
                                         ;; where the password file is stored
-  :site-title           "Smeagol"       ;; overall title of the site, used in
+  :site-title           "Smeagol"}       ;; overall title of the site, used in
                                         ;; page headings
-}

src/smeagol/include.clj

@@ -0,0 +1,61 @@
+(ns ^{:doc "Functions related to the include of markdown-paged in a given markdown."
+      :author "Michael Jerger"}  
+  smeagol.include
+  (:require
+    [clojure.string :as cs]
+    [schema.core :as s]
+    [com.stuartsierra.component :as component]
+    [smeagol.include.parse :as parse]
+    [smeagol.include.resolve :as resolve]
+    [smeagol.include.indent :as indent]))
+
+(s/defrecord Includer
+  [resolver])
+
+(defprotocol IncludeMd
+  (expand-include-md
+    [includer md-src]
+    "return a markdown containing resolved includes"))
+
+(s/defn
+  do-expand-one-include :- s/Str
+  [includer :- Includer
+   include :- parse/IncludeLink
+   md-src :- s/Str]
+  (let [{:keys [uri replace indent-heading indent-list]} include]
+    (cs/replace-first
+      md-src
+      (re-pattern (cs/escape
+                    replace
+                    {\[ "\\["
+                     \] "\\]"
+                     \( "\\("
+                     \) "\\)"}))
+      (indent/do-indent-list
+        indent-list
+        (indent/do-indent-heading
+          indent-heading
+          (resolve/resolve-md (:resolver includer) uri))))))
+
+(s/defn
+  do-expand-includes :- s/Str
+  [includer :- Includer
+   includes :- [parse/IncludeLink]
+   md-src :- s/Str]
+  (loop [loop-includes includes
+         result md-src]
+    (if (empty? loop-includes)
+      result
+      (recur
+        (rest loop-includes)
+        (do-expand-one-include includer (first loop-includes) result)))))
+
+(extend-type Includer
+  IncludeMd
+  (expand-include-md [includer md-src]
+    (do-expand-includes includer (parse/parse-include-md md-src) md-src)))
+
+(s/defn
+  new-includer
+  []
+  (map->Includer {}))

src/smeagol/include/indent.clj

@@ -0,0 +1,58 @@
+(ns ^{:doc "Functions related to the include of markdown-paged - handling the 
+list & heading indents of includes. This namespaces is implementation detail for
+smeagol.include and not inteded for direct usage."
+      :author "Michael Jerger"}
+  smeagol.include.indent
+  (:require
+    [clojure.string :as cs]
+    [schema.core :as s]))
+
+(s/defn
+  parse-list
+  [md-resolved :- s/Str]
+  (distinct
+    (into
+      (re-seq #"((^|\R? *)([\*\+-] ))" md-resolved)
+      (re-seq  #"((^|\R? *)([0-9]+\. ))" md-resolved))))
+
+(s/defn
+  parse-heading
+  [md-resolved :- s/Str]
+  (distinct
+    (re-seq #"((^|\R?)(#+ ))" md-resolved)))
+
+(s/defn
+  do-indent :- s/Str
+  [indent :- s/Num
+   indentor :- s/Str
+   elements
+   md-resolved :- s/Str]
+  (loop [result md-resolved
+         elements elements]
+    (if (empty? elements)
+      result
+      (let [element (first elements)
+            replace (nth element 1)
+            start (nth element 2)
+            end (nth element 3)]
+        (recur
+          (cs/replace
+           result
+           (re-pattern (cs/escape
+                         replace
+                         {\* "\\*"
+                          \n "\\n"}))
+           (str start (apply str (repeat indent indentor)) end))
+          (rest elements))))))
+
+(s/defn
+  do-indent-heading :- s/Str
+  [indent :- s/Num
+   md-resolved :- s/Str]
+  (do-indent indent "#" (parse-heading md-resolved) md-resolved))
+
+(s/defn
+  do-indent-list :- s/Str
+  [indent :- s/Num
+   md-resolved :- s/Str]
+  (do-indent indent " " (parse-list md-resolved) md-resolved))

src/smeagol/include/parse.clj

@@ -0,0 +1,50 @@
+(ns ^{:doc "Functions related to the include of markdown-paged - parsing of
+include links. This namespaces is implementation detail for 
+smeagol.include and not inteded for direct usage."
+      :author "Michael Jerger"}
+  smeagol.include.parse
+  (:require
+    [schema.core :as s]))
+
+(def IncludeLink
+  {:replace s/Str
+   :uri s/Str
+   :indent-heading s/Num
+   :indent-list s/Num})
+
+(s/defn
+  convert-indent-to-int :- s/Num
+  [indents :- [s/Str]]
+  (if (some? indents)
+      (Integer/valueOf (nth indents 2))
+      0))
+
+(s/defn
+  parse-indent-list
+  [md-src :- s/Str]
+  (re-matches #".*(:indent-list (\d)).*" md-src))
+
+(s/defn
+  parse-indent-heading
+  [md-src :- s/Str]
+  (re-matches #".*(:indent-heading (\d)).*" md-src))
+
+(s/defn
+  parse-include-link
+  [md-src :- s/Str]
+  (re-seq #".*(&\[\w*(.*)\w*\]\((.*)\)).*" md-src))
+
+(s/defn
+  parse-include-md :- [IncludeLink]
+  [md-src :- s/Str]
+  (vec
+    (map
+      (fn [parse-element]
+        (let [replace (nth parse-element 1)
+              uri (nth parse-element 3)
+              indents-text (nth parse-element 2)]
+          {:replace replace
+           :uri uri
+           :indent-heading (convert-indent-to-int (parse-indent-heading indents-text))
+           :indent-list (convert-indent-to-int (parse-indent-list indents-text))}))
+      (parse-include-link md-src))))

src/smeagol/include/resolve.clj

@@ -0,0 +1,46 @@
+(ns ^{:doc "Functions related to the include of markdown-paged - providing
+a plugable load-content componet. This namespaces is implementation detail for
+smeagol.include and not inteded for direct usage."
+      :author "Michael Jerger"}
+  smeagol.include.resolve
+  (:require
+    [schema.core :as s]
+    [com.stuartsierra.component :as component]))
+
+(s/defrecord Resolver
+  [type :- s/Keyword
+   local-base-dir :- s/Str])
+
+;As schema doesn't support s/defprotocol we use the dispatcher for annotation & validation.
+(s/defn dispatch-by-resolver-type :- s/Keyword
+  "Dispatcher for different resolver implementations."
+  [resolver :- Resolver
+   uri :- s/Str]
+  (:type resolver))
+
+(defmulti do-resolve-md
+  "Multimethod return a markdown file content for given uri."
+  dispatch-by-resolver-type)
+(s/defmethod do-resolve-md :default
+  [resolver :- Resolver
+   uri :- s/Str]
+  (throw (Exception. (str "No implementation for " resolver))))
+
+(defprotocol ResolveMd
+  (resolve-md
+    [resolver uri]
+    "return a markfown file content for given uri."))
+
+(extend-type Resolver
+  ResolveMd
+  (resolve-md [resolver uri]
+    (s/validate s/Str uri)
+    (s/validate s/Str (do-resolve-md resolver uri))))
+
+(s/defn
+  new-resolver
+  ([type :- s/Keyword]
+   (map->Resolver {:type type :local-base-dir nil}))
+  ([type :- s/Keyword
+    local-base-dir :- s/Str]
+   (map->Resolver {:type type :local-base-dir local-base-dir})))

src/smeagol/include/resolve_local_file.clj

@@ -0,0 +1,27 @@
+(ns ^{:doc "Functions related to the include of markdown-paged - providing
+a plugable load-local-include-links componet. This namespaces is implementation detail for
+smeagol.include and not inteded for direct usage."
+      :author "Michael Jerger"}
+  smeagol.include.resolve-local-file
+  (:require
+    [schema.core :as s]
+    [smeagol.include.resolve :as resolve]
+    [com.stuartsierra.component :as component]
+    [clojure.java.io :as cjio]
+    [taoensso.timbre :as timbre]))
+
+(s/defmethod resolve/do-resolve-md :local-file
+  [resolver
+   uri :- s/Str]
+  (let [file-name uri
+        file-path (cjio/file (:local-base-dir resolver) file-name)
+        exists? (.exists (clojure.java.io/as-file file-path))]
+    (cond exists?
+          (do
+            (timbre/info (format "Including page '%s' from file '%s'" uri file-path))
+            (slurp file-path)))))
+
+(s/defn
+  new-resolver
+  [local-base-dir :- s/Str]
+  (resolve/new-resolver :local-file local-base-dir))

src/smeagol/routes/wiki.clj

@@ -20,7 +20,10 @@
             [smeagol.sanity :refer [show-sanity-check-error]]
             [smeagol.util :as util]
             [smeagol.uploads :as ul]
-            [taoensso.timbre :as timbre]))
+            [taoensso.timbre :as timbre]
+            [com.stuartsierra.component :as component]
+            [smeagol.include.resolve-local-file :as resolve]
+            [smeagol.include :as include]))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;
@@ -108,13 +111,21 @@
   (edit-page request "stylesheet" ".css" "edit-css.html" "_edit-side-bar.md"))
 
 
+(def md-include-system
+  (component/start
+    (component/system-map
+      :resolver (resolve/new-resolver util/content-dir)
+      :includer (component/using
+                  (include/new-includer)
+                  [:resolver]))))
+
 (defn wiki-page
   "Render the markdown page specified in this `request`, if any. If none found, redirect to edit-page"
   [request]
   (or
     (show-sanity-check-error)
     (let [params (keywordize-keys (:params request))
-          page (or (:page params) (util/get-message :default-page-title "Introduction" request))
+          page (or (:page params) util/start-page (util/get-message :default-page-title "Introduction" request))
           file-name (str page ".md")
           file-path (cjio/file util/content-dir file-name)
           exists? (.exists (clojure.java.io/as-file file-path))]
@@ -125,7 +136,10 @@
                              (merge (util/standard-params request)
                                     {:title page
                                      :page page
-                                     :content (md->html (slurp file-path))
+                                     :content (md->html
+                                                (include/expand-include-md
+                                                  (:includer md-include-system)
+                                                  (slurp file-path)))
                                      :editable true})))
             true (response/redirect (str "/edit?page=" page))))))
 

src/smeagol/util.clj

@@ -35,6 +35,9 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 
+(def start-page
+  (:start-page  config))
+
 (def content-dir
   (or
     (:content-dir config)

test/smeagol/test/include.clj

@@ -0,0 +1,106 @@
+(ns smeagol.test.include
+  (:require [clojure.test :refer :all]
+            [schema.core :as s]
+            [com.stuartsierra.component :as component]
+            [smeagol.include.resolve :as resolve]
+            [smeagol.include :as sut]))
+
+(def include-simple
+  "# Heading1
+&[](./simple.md)")
+
+(def include-surounding-simple
+  "# Heading1
+Some surounding &[](./simple.md) text")
+
+(def include-heading-0
+  "# Heading1
+&[:indent-heading 0](./with-heading.md)")
+
+(def include-heading-list-1
+  "# Heading1
+&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)")
+
+(def include-heading-list-0
+  "# Heading1
+&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)")
+
+(def include-invalid-indent
+  "# Heading1
+&[ invalid input should default to indent 0 ](./simple.md)")
+
+(def include-spaced-indent
+  "# Heading1
+&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)")
+
+(def multi
+  "# Heading1
+&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)
+some text
+&[](./simple.md)
+more text.")
+
+(s/defmethod resolve/do-resolve-md :test-mock
+  [resolver
+   uri :- s/Str]
+  (cond
+    (= uri "./simple.md") "Simple content."
+    (= uri "./with-heading-and-list.md") "# Heading2
+some text
+* List
+
+## Heading 3
+more text"))
+
+
+
+(def system-under-test
+  (component/start
+    (component/system-map
+      :resolver (resolve/new-resolver :test-mock)
+      :includer (component/using
+                  (sut/new-includer)
+                  [:resolver]))))
+
+(deftest test-expand-include-md
+  (testing "The whole integration of include"
+    (is
+      (= "# Heading"
+         (sut/expand-include-md (:includer system-under-test) "# Heading")))
+    (is
+      (= "# Heading1
+Simple content."
+         (sut/expand-include-md
+           (:includer system-under-test)
+           include-simple)))
+    (is
+      (= "# Heading1
+Some surounding Simple content. text"
+         (sut/expand-include-md
+           (:includer system-under-test)
+           include-surounding-simple)))
+    (is
+      (= "# Heading1
+# Heading2
+some text
+* List
+
+## Heading 3
+more text"
+         (sut/expand-include-md
+           (:includer system-under-test)
+           include-heading-list-0)))
+    (is
+      (= "# Heading1
+### Heading2
+some text
+   * List
+
+#### Heading 3
+more text
+some text
+Simple content.
+more text."
+        (sut/expand-include-md
+          (:includer system-under-test)
+          multi)))))

test/smeagol/test/include/indent.clj

@@ -0,0 +1,35 @@
+(ns smeagol.test.include.indent
+  (:require [clojure.test :refer :all]
+            [smeagol.include.indent :as sut]))
+
+(deftest test-parse-heading
+  (testing
+    (is (= '(["# " "# " "" "# "])
+           (sut/parse-heading "# h1")))
+    (is (= '(["\n# " "\n# " "\n" "# "])
+           (sut/parse-heading "\n# h1")))))
+
+(deftest test-indent-heading
+  (testing
+    (is (= "# h1"
+          (sut/do-indent-heading 0 "# h1")))
+    (is (= "### h1"
+          (sut/do-indent-heading 2 "# h1")))
+    (is (= "\n### h1"
+          (sut/do-indent-heading 2 "\n# h1")))))
+
+(deftest test-parse-list
+  (testing
+    (is (= '([" * " " * " " " "* "])
+           (sut/parse-list " * list")))
+    (is (= '(["\n * " "\n * " "\n " "* "])
+           (sut/parse-list "\n * list")))))
+
+(deftest test-indent-list
+  (testing
+    (is (= " * list"
+          (sut/do-indent-list 0 " * list")))
+    (is (= "   * list"
+          (sut/do-indent-list 2 " * list")))
+    (is (= "\n   * list"
+          (sut/do-indent-list 2 "\n * list")))))

test/smeagol/test/include/parse.clj

@@ -0,0 +1,91 @@
+(ns smeagol.test.include.parse
+  (:require [clojure.test :refer :all]
+            [schema.core :as s]
+            [smeagol.include.parse :as sut]))
+
+(def include-simple
+  "# Heading1
+&[](./simple.md)")
+
+(def include-surounding-simple
+  "# Heading1
+Some surounding &[](./simple.md) text")
+
+(def include-heading-0
+  "# Heading1
+&[:indent-heading 0](./with-heading.md)")
+
+(def include-heading-list-1
+  "# Heading1
+&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)")
+
+(def include-heading-list-0
+  "# Heading1
+&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)")
+
+(def include-invalid-indent
+  "# Heading1
+&[ invalid input should default to indent 0 ](./simple.md)")
+
+(def include-spaced-indent
+  "# Heading1
+&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)")
+
+(def multi
+  "# Heading1
+&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)
+some text
+&[](./simple.md)
+more text.")
+
+
+(deftest test-parse-include-md
+  (testing "parse include links"
+    (is
+      (= []
+         (sut/parse-include-md "# Heading")))
+    (is
+      (= [{:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           include-simple)))
+    (is
+      (= [{:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           include-surounding-simple)))
+    (is
+      (= [{:replace "&[:indent-heading 0](./with-heading.md)" :uri "./with-heading.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           include-heading-0)))
+    (is
+      (= [{:replace
+            "&[:indent-heading 1 :indent-list 1](./with-heading-and-list.md)"
+           :uri "./with-heading-and-list.md", :indent-heading 1, :indent-list 1}]
+         (sut/parse-include-md
+           include-heading-list-1)))
+    (is
+      (= [{:replace
+            "&[:indent-list 0 :indent-heading 0](./with-heading-and-list.md)"
+           :uri "./with-heading-and-list.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           include-heading-list-0)))
+    (is
+      (= [{:replace
+            "&[ invalid input should default to indent 0 ](./simple.md)"
+           :uri "./simple.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           include-invalid-indent)))
+    (is
+      (= [{:replace
+            "&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)"
+           :uri "./with-heading-and-list.md", :indent-heading 2, :indent-list 3}]
+         (sut/parse-include-md
+           include-spaced-indent)))
+    (is
+      (= [{:replace
+            "&[ :indent-heading 2   :indent-list 33  ](./with-heading-and-list.md)"
+           :uri "./with-heading-and-list.md",
+           :indent-heading 2,
+           :indent-list 3}
+          {:replace "&[](./simple.md)" :uri "./simple.md", :indent-heading 0, :indent-list 0}]
+         (sut/parse-include-md
+           multi)))))

test/smeagol/test/include/resolve.clj

@@ -0,0 +1,8 @@
+(ns smeagol.test.include.resolve
+  (:require [clojure.test :refer :all]
+            [smeagol.include.resolve :as sut]))
+
+(deftest test-local-links
+  (testing "Rewriting of local links"
+    (is (thrown? Exception
+          (sut/resolve-md (sut/new-resolver (:default)) "./some-uri.md")))))