From bd0e0a802afa024e205c7ef078a70be8eb469a3c Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Mon, 24 Jul 2017 00:31:11 +0100 Subject: [PATCH 01/14] WIP on Illuminator: minor look and feel tweeks --- resources/public/content/_side-bar.md | 2 +- resources/public/content/stylesheet.css | 1 + resources/templates/base.html | 2 +- src/smeagol/history.clj | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/public/content/_side-bar.md b/resources/public/content/_side-bar.md index 321efc1..d3f2a83 100644 --- a/resources/public/content/_side-bar.md +++ b/resources/public/content/_side-bar.md @@ -1,3 +1,3 @@ This is the side bar. There's nothing in it yet. You could [edit](edit?page=_side-bar) it to provide internal navigation or branding. -If you don't like it on the left, float it to the right (or do something entirely different) by editing the [stylesheet](/edit-css?page=stylesheet). +If you don't like it on the left, float it to the right (or do something entirely different) by editing the [stylesheet](edit-css?page=stylesheet). diff --git a/resources/public/content/stylesheet.css b/resources/public/content/stylesheet.css index a9d64f9..b31f747 100644 --- a/resources/public/content/stylesheet.css +++ b/resources/public/content/stylesheet.css @@ -74,6 +74,7 @@ form { header { margin-top: 0; width:100%; + max-width: 100%; background-color: gray; color: white; } diff --git a/resources/templates/base.html b/resources/templates/base.html index c6c5f56..38a565d 100644 --- a/resources/templates/base.html +++ b/resources/templates/base.html @@ -79,7 +79,7 @@
one wiki to rule them allOne Wiki to rule them all || Smeagol wiki engine {{version}} || - The Web Engineering Factory & Toolworks Developed by WEFT + The Web Engineering Factory & Toolworks Developed by WEFT
Built with LuminusWeb || diff --git a/src/smeagol/history.clj b/src/smeagol/history.clj index 134701d..ceff181 100644 --- a/src/smeagol/history.clj +++ b/src/smeagol/history.clj @@ -49,9 +49,10 @@ log-entry)) -(defn find-history [^String git-directory-path ^String file-path] +(defn find-history "Return the log entries in the repository at this `git-directory-path` which refer to changes to the file at this `file-path`." + [^String git-directory-path ^String file-path] (let [repository (git/load-repo git-directory-path)] (filter #(entry-contains % file-path) From 7fb6bb19baaf5d8dfc4bf164cc1431856dc8f888 Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Mon, 24 Jul 2017 06:40:31 +0100 Subject: [PATCH 02/14] Ignore all Bower and Vendor files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b510bc5..77a34bc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ pom.xml.asc .lein-failures .lein-env + +/node_modules/ +/resources/public/vendor/ From fa944f17803ab682ccd6f8701d0f3487d7f0f7b7 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 26 Jul 2017 14:50:17 +0100 Subject: [PATCH 03/14] #13 WOW! It works! And it's beautiful! --- README.md | 210 +++++++++++++---------- project.clj | 36 ++-- resources/public/content/Introduction.md | 33 +++- resources/public/content/stylesheet.css | 9 + resources/public/data/london.csv | 158 +++++++++++++++++ resources/public/data/london.png | Bin 0 -> 6863 bytes resources/templates/base.html | 1 - resources/templates/wiki.html | 13 ++ src/smeagol/formatting.clj | 133 ++++++++++++++ src/smeagol/routes/wiki.clj | 26 ++- src/smeagol/util.clj | 38 +--- 11 files changed, 498 insertions(+), 159 deletions(-) create mode 100644 resources/public/data/london.csv create mode 100644 resources/public/data/london.png create mode 100644 src/smeagol/formatting.clj diff --git a/README.md b/README.md index 88de98d..7c88309 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,119 @@ -![One wiki to rule them all](https://www.weft.scot/images/smeagol.png) - -# Welcome to Smeagol! -Smeagol is a simple Wiki engine inspired by [Gollum](https://github.com/gollum/gollum/wiki). Gollum is a Wiki engine written in Ruby, which uses a number of simple text formats including [Markdown](http://daringfireball.net/projects/markdown/), and which uses [Git](http://git-scm.com/) to provide versioning and backup. I needed a new Wiki for a project and thought Gollum would be ideal - but unfortunately it doesn't provide user authentication, which I needed, and it was simpler for me to reimplement the bits I did need in Clojure than to modify Gollum. - -So at this stage Smeagol is a Wiki engine written in Clojure which uses Markdown as its text format, which does have user authentication, and which uses Git as its versioning and backup system. - -## Status -Smeagol is now a fully working small Wiki engine, and meets my own immediate needs. - -## Markup syntax -Smeagol uses the Markdown format as provided by [markdown-clj](https://github.com/yogthos/markdown-clj), with the addition that anything enclosed in double square brackets, \[\[like this\]\], will be treated as a link into the wiki itself. - -## Security and authentication -Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: - - {:admin {:password "admin" :email "admin@localhost" :admin true} - :adam {:password "secret" :email "adam@localhost"}} - -that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. - -## Images -Smeagol does not currently have any mechanism to upload images. You can, however, link to images already available on the web, like this: - -![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) - -## Todo -* Mechanism to add users through the user interface; - -## Advertisement -If you like what you see here, I am available for work on open source Clojure projects. Contact me via [WEFT](http://www.weft.scot/). - -### Phoning home -Smeagol currently requests the WEFT logo in the page footer from my home site. This is mainly so I can get a feel for how many people are using the product. If you object to this, edit the file - - resources/templates/base.html - -and replace the line - - The Web Engineering Factory & Toolworks Developed by WEFT - -with the line - - The Web Engineering Factory & Toolworks Developed by WEFT - -## License -Copyright © 2014-2015 Simon Brooke. Licensed under the GNU General Public License, -version 2.0 or (at your option) any later version. If you wish to incorporate -parts of Smeagol into another open source project which uses a less restrictive -license, please contact me; I'm open to dual licensing it. - -## Prerequisites -You will need [Leiningen](https://github.com/technomancy/leiningen) 2.0 or above installed. - -You will need [node](https://nodejs.org/en/) and [bower](https://bower.io/) installed. - -## Running -To start a web server for the application, run: - - lein bower install - lein ring server - -Alternatively, if you want to deploy to a servlet container (which I would strongly recommend), the simplest thing is to run: - - lein bower install - lein ring uberwar - -(a command which I'm sure Smeagol would entirely appreciate) and deploy the resulting war file. - -## Experimental Docker image - -You can now run Smeagol as a [Docker](http://www.docker.com) image. To run my Docker image, use - - docker run simonbrooke/smeagol - -Smeagol will run, obviously, on the IP address of your Docker image, on port 8080. To find the IP address, start the image using the command above and then use - - docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(docker ps -q) - -Suppose this prints '10.10.10.10', then the URL to browse to will be http://10.10.10.10:8080/smeagol/ - -This image is _experimental_, but it does seem to work fairly well. What it does **not** yet do, however, is push the git repository to a remote location, so when you tear the Docker image down your edits will be lost. My next objective for this image is for it to have a cammand line parameter being the git address of a repository from which it can initialise the Wiki content, and to which it will periodically push local changes to the Wiki content. - -To build your own Docker image, run: - - lein clean - lein bower install - lein ring uberwar - lein docker build - -This will build a new Docker image locally; you can, obviously, push it to your own Docker repository if you wish. +![One wiki to rule them all](https://www.weft.scot/images/smeagol.png) + +# Welcome to Smeagol! +Smeagol is a simple Wiki engine inspired by [Gollum](https://github.com/gollum/gollum/wiki). Gollum is a Wiki engine written in Ruby, which uses a number of simple text formats including [Markdown](http://daringfireball.net/projects/markdown/), and which uses [Git](http://git-scm.com/) to provide versioning and backup. I needed a new Wiki for a project and thought Gollum would be ideal - but unfortunately it doesn't provide user authentication, which I needed, and it was simpler for me to reimplement the bits I did need in Clojure than to modify Gollum. + +So at this stage Smeagol is a Wiki engine written in Clojure which uses Markdown as its text format, which does have user authentication, and which uses Git as its versioning and backup system. + +## Status +Smeagol is now a fully working small Wiki engine, and meets my own immediate needs. + +## Markup syntax +Smeagol uses the Markdown format as provided by [markdown-clj](https://github.com/yogthos/markdown-clj), with the addition that anything enclosed in double square brackets, \[\[like this\]\], will be treated as a link into the wiki itself. + +## Security and authentication +Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: + + {:admin {:password "admin" :email "admin@localhost" :admin true} + :adam {:password "secret" :email "adam@localhost"}} + +that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. + +## Images +Smeagol does not currently have any mechanism to upload images. You can, however, link to images already available on the web, like this: + +![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) + +## Now with data visualisation + +Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), you can now embed visualisations into Smeagol pages, like this: + +### Flight punctuality at London airports + +Example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): +```vis +data: + url: "data/london.csv" +transform: + - + filter: datum.year == 2016 +mark: rect +encoding: + x: + type: nominal + field: source + y: + type: nominal + field: dest + color: + type: quantitative + field: flights + aggregate: sum +``` + +Note that this visualisation will not be rendered in the GitHub wiki, as it doesn't have Smeagol's data visualisation magic. This is what it should look like: + +![Example visualisation](data/london.png) + +## Advertisement +If you like what you see here, I am available for work on open source Clojure projects. + +### Phoning home +Smeagol currently requests the WEFT logo in the page footer from my home site. This is mainly so I can get a feel for how many people are using the product. If you object to this, edit the file + + resources/templates/base.html + +and replace the line + + The Web Engineering Factory & Toolworks Developed by WEFT + +with the line + + The Web Engineering Factory & Toolworks Developed by WEFT + +## License +Copyright © 2014-2015 Simon Brooke. Licensed under the GNU General Public License, +version 2.0 or (at your option) any later version. If you wish to incorporate +parts of Smeagol into another open source project which uses a less restrictive +license, please contact me; I'm open to dual licensing it. + +## Prerequisites +You will need [Leiningen](https://github.com/technomancy/leiningen) 2.0 or above installed. + +You will need [node](https://nodejs.org/en/) and [bower](https://bower.io/) installed. + +## Running +To start a web server for the application, run: + + lein bower install + lein ring server + +Alternatively, if you want to deploy to a servlet container (which I would strongly recommend), the simplest thing is to run: + + lein bower install + lein ring uberwar + +(a command which I'm sure Smeagol would entirely appreciate) and deploy the resulting war file. + +## Experimental Docker image + +You can now run Smeagol as a [Docker](http://www.docker.com) image. To run my Docker image, use + + docker run simonbrooke/smeagol + +Smeagol will run, obviously, on the IP address of your Docker image, on port 8080. To find the IP address, start the image using the command above and then use + + docker inspect --format '{{ .NetworkSettings.IPAddress }}' $(docker ps -q) + +Suppose this prints '10.10.10.10', then the URL to browse to will be http://10.10.10.10:8080/smeagol/ + +This image is _experimental_, but it does seem to work fairly well. What it does **not** yet do, however, is push the git repository to a remote location, so when you tear the Docker image down your edits will be lost. My next objective for this image is for it to have a cammand line parameter being the git address of a repository from which it can initialise the Wiki content, and to which it will periodically push local changes to the Wiki content. + +To build your own Docker image, run: + + lein clean + lein bower install + lein ring uberwar + lein docker build + +This will build a new Docker image locally; you can, obviously, push it to your own Docker repository if you wish. diff --git a/project.clj b/project.clj index 40eeea6..347c7ed 100644 --- a/project.clj +++ b/project.clj @@ -1,30 +1,30 @@ -(defproject smeagol "0.5.0-rc3" +(defproject smeagol "0.5.1-SNAPSHOT" :description "A simple Git-backed Wiki inspired by Gollum" :url "https://github.com/simon-brooke/smeagol" - :dependencies [[org.clojure/clojure "1.7.0"] + :dependencies [[org.clojure/clojure "1.8.0"] [org.clojure/core.memoize "0.5.9"] - [com.taoensso/encore "2.91.1"] - [lib-noir "0.9.9" :exclusions [org.clojure/tools.reader]] - [com.cemerick/url "0.1.1"] - [ring-server "0.4.0"] - [selmer "1.10.9"] - + [org.clojure/data.json "0.2.6"] [org.clojure/tools.logging "0.4.0"] - [com.taoensso/timbre "4.10.0"] + [clj-jgit "0.8.9"] + [clj-yaml "0.4.0"] + [com.cemerick/url "0.1.1"] [com.fzakaria/slf4j-timbre "0.3.7"] + [com.taoensso/encore "2.91.1"] + [com.taoensso/timbre "4.10.0"] + [com.taoensso/tower "3.0.2" :exclusions [com.taoensso/encore]] + [crypto-password "0.2.0"] + [environ "1.1.0"] + [im.chit/cronj "1.4.4"] + [lib-noir "0.9.9" :exclusions [org.clojure/tools.reader]] + [markdown-clj "0.9.99" :exclusions [com.keminglabs/cljx]] + [noir-exception "0.2.5"] [org.slf4j/slf4j-api "1.7.25"] [org.slf4j/log4j-over-slf4j "1.7.25"] [org.slf4j/jul-to-slf4j "1.7.25"] [org.slf4j/jcl-over-slf4j "1.7.25"] - - [com.taoensso/tower "3.0.2" :exclusions [com.taoensso/encore]] - [markdown-clj "0.9.99" :exclusions [com.keminglabs/cljx]] - [crypto-password "0.2.0"] - [clj-jgit "0.8.9"] - [environ "1.1.0"] - [im.chit/cronj "1.4.4"] - [noir-exception "0.2.5"] - [prone "1.1.4"]] + [prone "1.1.4"] + [ring-server "0.4.0"] + [selmer "1.10.9"]] :repl-options {:init-ns smeagol.repl} :jvm-opts ["-server"] diff --git a/resources/public/content/Introduction.md b/resources/public/content/Introduction.md index 18df529..7c88309 100644 --- a/resources/public/content/Introduction.md +++ b/resources/public/content/Introduction.md @@ -24,8 +24,39 @@ Smeagol does not currently have any mechanism to upload images. You can, however ![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) +## Now with data visualisation + +Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), you can now embed visualisations into Smeagol pages, like this: + +### Flight punctuality at London airports + +Example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): +```vis +data: + url: "data/london.csv" +transform: + - + filter: datum.year == 2016 +mark: rect +encoding: + x: + type: nominal + field: source + y: + type: nominal + field: dest + color: + type: quantitative + field: flights + aggregate: sum +``` + +Note that this visualisation will not be rendered in the GitHub wiki, as it doesn't have Smeagol's data visualisation magic. This is what it should look like: + +![Example visualisation](data/london.png) + ## Advertisement -If you like what you see here, I am available for work on open source Clojure projects. Contact me via [WEFT](http://www.weft.scot/). +If you like what you see here, I am available for work on open source Clojure projects. ### Phoning home Smeagol currently requests the WEFT logo in the page footer from my home site. This is mainly so I can get a feel for how many people are using the product. If you object to this, edit the file diff --git a/resources/public/content/stylesheet.css b/resources/public/content/stylesheet.css index b31f747..e8c2a16 100644 --- a/resources/public/content/stylesheet.css +++ b/resources/public/content/stylesheet.css @@ -197,6 +197,11 @@ th { border: thin solid silver; } + +.data-visualisation { + padding: 0.25em 5%; +} + .error { width: 100%; background-color: red; @@ -231,6 +236,10 @@ th { color: white; } +.vega-bindings, .vega-actions { + font-size: 66%; +} + .warn { color: maroon; } diff --git a/resources/public/data/london.csv b/resources/public/data/london.csv new file mode 100644 index 0000000..cf1dc34 --- /dev/null +++ b/resources/public/data/london.csv @@ -0,0 +1,158 @@ +"source","dest","airline","flights","onTimePerf","delayAverage","year" +"LHR","ORD","AA",2490,66.33,21.11,2010 +"LHR","ORD","BA",1413,57.63,23.3,2010 +"LHR","ORD","UA",2105,73.24,14.57,2010 +"LHR","ORD","VS",218,77.06,11.1,2010 +"LHR","LAX","AA",706,66.38,17.66,2010 +"LHR","LAX","BA",1914,50.18,31.01,2010 +"LHR","LAX","UA",698,81.81,13.62,2010 +"LHR","LAX","VS",1285,56.86,21.92,2010 +"LHR","JFK","AA",3205,67.36,20.31,2010 +"LHR","JFK","BA",4181,66.59,20.84,2010 +"LHR","JFK","DL",1611,62.12,23.74,2010 +"LHR","JFK","KU",305,46.56,38.75,2010 +"LHR","JFK","VS",2067,59.62,23.69,2010 +"LHR","EWR","BA",1828,70.25,17.67,2010 +"LHR","EWR","CO",2697,69.28,18.87,2010 +"LHR","EWR","VS",1399,65.62,23.15,2010 +"LHR","IAD","BA",2011,62.11,21.71,2010 +"LHR","IAD","UA",2094,74.67,14.11,2010 +"LHR","IAD","VS",700,66.24,19.75,2010 +"LCY","JFK","BA",1012,89.12,5.67,2010 +"LHR","ORD","AA",2532,76.87,13.34,2011 +"LHR","ORD","BA",1440,76.32,14.42,2011 +"LHR","ORD","UA",2106,79.36,15.2,2011 +"LHR","ORD","VS",302,82.72,8.66,2011 +"LHR","LAX","AA",730,75.48,13.66,2011 +"LHR","LAX","BA",1984,63.46,18.47,2011 +"LHR","LAX","UA",698,83.52,11.7,2011 +"LHR","LAX","VS",1397,79,11.2,2011 +"LHR","JFK","AA",2917,75.8,14.77,2011 +"LHR","JFK","BA",4944,73.56,14.65,2011 +"LHR","JFK","DL",2144,80.91,13.05,2011 +"LHR","JFK","KU",302,52.65,30.02,2011 +"LHR","JFK","VS",2104,76.32,15.09,2011 +"LHR","EWR","BA",2129,80.7,11.24,2011 +"LHR","EWR","CO",3503,78.42,14.19,2011 +"LHR","EWR","VS",1410,77.52,13.39,2011 +"LHR","IAD","BA",2157,74.08,14.11,2011 +"LHR","IAD","UA",2516,80.79,14,2011 +"LHR","IAD","VS",711,82.42,9.73,2011 +"LCY","JFK","BA",1031,92.05,4.12,2011 +"LHR","ORD","AA",2523,72.07,20.03,2012 +"LHR","ORD","BA",1445,70.03,17.73,2012 +"LHR","ORD","UA",2102,76.08,15.35,2012 +"LHR","ORD","VS",308,83.77,8.31,2012 +"LHR","LAX","AA",714,75.35,12.03,2012 +"LHR","LAX","BA",1874,63.48,18.34,2012 +"LHR","LAX","UA",693,76.16,14.88,2012 +"LHR","LAX","VS",1324,76.13,14.08,2012 +"LHR","JFK","AA",3367,75.72,12.75,2012 +"LHR","JFK","BA",5129,69.28,17.68,2012 +"LHR","JFK","DL",2151,77.63,13.59,2012 +"LHR","JFK","KU",307,52.12,32.02,2012 +"LHR","JFK","VS",2227,81.9,11.32,2012 +"LHR","EWR","BA",1920,73.02,15.88,2012 +"LHR","EWR","CO",612,84.48,9.24,2012 +"LHR","EWR","UA",2872,69.76,17.91,2012 +"LHR","EWR","VS",1421,76.14,16.68,2012 +"LHR","IAD","BA",2014,71.45,16.23,2012 +"LHR","IAD","UA",2444,69.04,18.98,2012 +"LHR","IAD","VS",703,87.06,7.71,2012 +"LCY","JFK","BA",474,91.33,3.74,2012 +"LCY","JFK","BA",537,89.37,4.71,2012 +"LHR","ORD","AA",2513,74.69,15.29,2013 +"LHR","ORD","BA",1440,65.35,20.84,2013 +"LHR","ORD","UA",2110,78.39,14.13,2013 +"LHR","ORD","VS",324,83.02,9.05,2013 +"LHR","LAX","AA",728,76.48,12.56,2013 +"LHR","LAX","BA",1864,59.5,27.43,2013 +"LHR","LAX","UA",702,85.47,9.86,2013 +"LHR","LAX","VS",1288,79,13.76,2013 +"LHR","JFK","AA",2991,76.4,13.73,2013 +"LHR","JFK","BA",5350,66.97,18.95,2013 +"LHR","JFK","DL",2166,82.09,10.81,2013 +"LHR","JFK","KU",306,62.42,21.53,2013 +"LHR","JFK","VS",2825,81.91,10.53,2013 +"LHR","EWR","BA",1838,71.44,16.72,2013 +"LHR","EWR","UA",3494,80.59,12.36,2013 +"LHR","EWR","VS",1419,81.66,11.7,2013 +"LHR","IAD","BA",2012,71.02,18.67,2013 +"LHR","IAD","UA",2496,80.39,12.82,2013 +"LHR","IAD","VS",702,86.89,7.1,2013 +"LCY","JFK","BA",945,90.64,4.58,2013 +"LGW","LAX","DY",105,72.38,36.83,2014 +"LGW","JFK","DY",156,51.92,27.79,2014 +"LHR","ORD","AA",2476,73.5,22.09,2014 +"LHR","ORD","BA",1444,65.28,18.7,2014 +"LHR","ORD","UA",2110,77.56,13.68,2014 +"LHR","ORD","VS",348,81.32,8.86,2014 +"LHR","LAX","AA",728,76.79,18.56,2014 +"LHR","LAX","BA",1450,56.84,23.1,2014 +"LHR","LAX","DL",127,92.13,6.92,2014 +"LHR","LAX","UA",698,85.1,8.53,2014 +"LHR","LAX","VS",1217,80.99,12.58,2014 +"LHR","JFK","AA",2287,78.18,14.74,2014 +"LHR","JFK","BA",6018,71.29,16.59,2014 +"LHR","JFK","DL",2123,82.13,11.19,2014 +"LHR","JFK","KU",310,48.22,33.17,2014 +"LHR","JFK","VS",2820,79.77,12.45,2014 +"LHR","EWR","BA",1943,71.46,19.7,2014 +"LHR","EWR","UA",3485,78.3,15.19,2014 +"LHR","EWR","VS",1420,86.6,8.37,2014 +"LHR","IAD","BA",1807,74.58,15.31,2014 +"LHR","IAD","UA",2130,81.13,13.61,2014 +"LHR","IAD","VS",695,89.06,6.9,2014 +"LCY","JFK","BA",1003,91.23,4.43,2014 +"LGW","LAX","DY",328,66.46,21.41,2015 +"LGW","JFK","DY",543,64.09,31.78,2015 +"LHR","ORD","AA",2062,72.09,27.59,2015 +"LHR","ORD","BA",1442,71.4,16.24,2015 +"LHR","ORD","UA",2099,83.94,10.37,2015 +"LHR","ORD","VS",330,78.79,11.61,2015 +"LHR","LAX","AA",1265,75.42,14.5,2015 +"LHR","LAX","BA",1450,66.9,17.67,2015 +"LHR","LAX","DL",540,79.44,10.97,2015 +"LHR","LAX","UA",704,89.2,7.5,2015 +"LHR","LAX","VS",1230,80.81,12.74,2015 +"LHR","JFK","AA",2130,77.45,15.21,2015 +"LHR","JFK","BA",5994,77.84,13.61,2015 +"LHR","JFK","DL",2069,80.75,13.51,2015 +"LHR","JFK","KU",310,59.22,32.67,2015 +"LHR","JFK","VS",3363,76.35,13.82,2015 +"LHR","EWR","BA",1459,76.47,14.03,2015 +"LHR","EWR","DL",381,95.54,2.88,2015 +"LHR","EWR","UA",3493,79.23,16.31,2015 +"LHR","EWR","VS",871,81.75,10.7,2015 +"LHR","IAD","BA",1421,76.89,12.03,2015 +"LHR","IAD","UA",2126,81.74,14.02,2015 +"LHR","IAD","VS",703,85.49,12.59,2015 +"LCY","JFK","BA",937,92.74,4.52,2015 +"LTN","EWR","DJT",316,81.53,18.18,2015 +"LGW","LAX","DY",421,59.86,23.64,2016 +"LGW","LAX","DI",38,65.79,14.39,2016 +"LGW","JFK","BA",486,69.75,18.02,2016 +"LGW","JFK","DY",689,65.31,28.18,2016 +"LGW","JFK","DI",39,69.23,15.49,2016 +"LHR","ORD","AA",2453,71.82,26.15,2016 +"LHR","ORD","BA",1443,68.88,17.58,2016 +"LHR","ORD","UA",2085,82.25,14.2,2016 +"LHR","ORD","VS",333,89.79,5.07,2016 +"LHR","LAX","AA",1411,68.25,19.46,2016 +"LHR","LAX","BA",1452,54.34,24.89,2016 +"LHR","LAX","UA",700,87,8.48,2016 +"LHR","LAX","VS",1376,79.58,10.13,2016 +"LHR","JFK","AA",2623,76.82,21.41,2016 +"LHR","JFK","BA",5625,69.97,17.15,2016 +"LHR","JFK","DL",2081,80.37,13.42,2016 +"LHR","JFK","KU",253,19.76,71.96,2016 +"LHR","JFK","VS",3489,79.67,11.44,2016 +"LHR","EWR","AI",118,65.25,21.14,2016 +"LHR","EWR","BA",1444,76.11,13.49,2016 +"LHR","EWR","UA",3472,79.03,15.71,2016 +"LHR","EWR","VS",722,82.27,9.48,2016 +"LHR","IAD","BA",1415,71.59,17.44,2016 +"LHR","IAD","UA",2134,82.05,13.24,2016 +"LHR","IAD","VS",699,84.69,8.02,2016 +"LCY","JFK","BA",921,90.01,5.26,2016 +"LTN","EWR","DJT",333,87.05,8.44,2016 diff --git a/resources/public/data/london.png b/resources/public/data/london.png new file mode 100644 index 0000000000000000000000000000000000000000..8e28638ad65515329997efc3e79a8b7205e65ff9 GIT binary patch literal 6863 zcmaiZ2Q*w=+xB2E$}n2=GI}o&B}l^PL?{wG{T_`mNH3$TP>g#El zgFqx;;5-*X0UY5~gcT5ok7>5js0|usI;h;G2}iSUmOE7xGu*FFWH$nyFL9#`U!iBn{SYH zjB5D$$??Vy)j*TUGwaEVY!$0OikI%Flm++KZ72a5krj;#BS067f~dn`jA<6 zMvN9875wB9WXFtw7Q?+ix9km_9V|Wl?!9>x5}=Z0ve|yJ6(lYoRXC(V%9#j5yXLTr z1b%koa<`8!us}9D9Aq49Zd%;_=0B(#WUTU|VD-lSOg>BS6_SUt@Gs`H+k`kCldXiA zQGLj>;{i6v@)%woorQi6s-~oSv3|~@%n=-Xy2zw_rHOtANgTmH#+M_aXCpv%IE5!_>N3i=L6zbww&n~n0D}K5G+#bkua)GbVH5E zEq5rC9tW!;U!V&nbIf*vEcfC46-YOPR?P9U_|KS?;+aVf)-3EOTiI{+?=N7!(6B4P zVmntI;fq!86A7QNFk=#5u$|)KDc#O{G7&IbjXs!-?`{}xlV|=Zu$m?TA2N z3xKKcgna6!(mNqUAH1zI&mtB^N0tpy12sJ@xtInSQzMe>BEXw83tWf%C{Yv$wR97U zawf{-3HTTc4)oMk0BjXOd9eTL`};`zYoBJu{x&o#84g@Urp6W=nl$P!JnbB>Lu&zw5=e%U+JcsHtFeF3N2w@Erlyl*n0Gps*WQiIhITaz^ zMIT(B-gebQWmiw#U;3-#t+lFdA$*2DYs*gZ*x737__fpa+W5^V3$hKqqghs7@>T}a z@y+HvB#|A);C)txs}eNkBlTHqHFFHJ-#u7OfaUc9p8+#l?4+TWYi#Qg7vec2aV*_-#D7SfmeMagUdqNQleCFkaX^BU# z@Pm6;KB6aOHTeWxJYhxOn#3jYX!}X*aZ{o5Uze@a7&A5r5K&O z++Thiu*hCRXtXEO&CC)0A9qcP`Pvu4X7%}Q#*Zgun5+nct!n%xO%I)A&!HGr&`bJg z>(x)P1>zokqbTRDfgNLd%V}>xD_wVk~H_VedLsxJ~89 zd}Te4%WayQY~7*x1Dq#kD?G|)5CY|+j9yoGhUJ|pMEO`>^sCwpEOCh=#Q4SaY)-S| z!g@IKy5ch*$TQD|Dz=lPaP7f}l&?8^KdXKsy7I!u5)6%fbd2_Wl!QFke6%@~;Jf!+ zQOLb^8=3u|$ShSGTE#$~qkbMNQ7A#7%)d(eqJq8?khB)0Ayw|^_(*ZOKU3#5QzL{Z zHk1(o=d%?QrgDl)NL9p&MZVWQ094i~j#~gY5Z%V~A$3!f-(q3=_rma3ov|4TFP*hhl6uFdDn7aK8_&pO+ zl#;G(s_Fhl-hr!aK5lF3{`OI^QPSHvWhs8XrDGTuvK-cEJJb8-V@SjHpV?1s5$6MY z0AeGe`gC#eH2rC;6c7E+fSFG}0y-M%eQ`CTZS6;KZo8Hg=9=mJhC@v7&1Tfa>ZD5z zf?9#GiHLLI6H`7sd#X(;v4nJ3PuA9^(4b?5N3CB-6pRi#2*e`4*3=XEs`YY=DJ1%8 zxlSde`+`o9IffJx1*QUI)%u|LK?#>pAxtU(dC}H$5%_WYyn{bx_?6!?TCPnE(!i_F z2_4}{UHgm!}{*6TP{U1dhJe-+pCgp>l2 z@P;hj0}7`-b+Ks5eNEFa0Rx7QIQV%b18WL9GRG!yAUq-DA6C`#?rB1lmJZPO=~PUV zzj!#y!P5A#muNP3X4av10D907dD{68*Ct$Q3$ zoLVAXc75}mX}F7mv)c@wlC<;OqWzRB!jbi0eX#}5v2+cD9u#<@21Ellt5P~Kz6I@+ zPd=Cq7jOAt$vV3PbqLglD1aZP!bh&wA4zt3LN&gphc=Bl71Aj~lWO0>PAi>_yTX6) zOHk!%|6`kx0@j{i@%{O6pm}N6zyQmEXZ2w3=6KH}OEOIQh5DtFyqTc{;=52=#V?7O zUcM|e!h6T39?tvf^|H-Naoxx~jVZ11s=Y-`dKh!~Y> z%D|SkYhUErH{+h&bjuo5BW+7YgvN{^1HcWy;pQ8%SOWwbq^dz4;3U$@ zz|2|8QBwCopUskRUJC4O&V;N9BSJGrQLhn*2F0&xQmV!b-y}mpV1Ol1dBBt)_S5sC zRx<`Y1RD#1Ve7p-0yR5hom4ie^U<;MC@FCcP@N)Cf=o)$=5jTE(rcA`(tuY@!Zn{Yh)tqr{){BbjitgOi z&{mcivHcZGtfMG7Wymw_EZ9Qlg@@xosyjm%eCODw5@iB?GD1-AxGHTvEsS;KQKHId zXHC5+7O|v+z&ZGuGWVw6qRQoKBBvtUJDU2v*5ZeoC)={DtO@}yCjKNSIjFS#z7qCV z218)V%&U|8#0CZEa?>G1&hsXs7!gOt`*O+mm$g?@;5)7@>t0hV z(TMl?ABr|pHSVUVAw13C6NlqhJ!OxEuKTSgf3vUJIlND2b}Mv0*FIzZ<6L3-P>)3R zB6fkYw@?;t(|%KCb0{J)mO4an)s$$h0Ws!1UgbVyzxS}~l>!2lS4zB9{%5+O z)ge>H37?v4MeNIe6i++Qo%*MC#7BhC++4l=yo~-%F)x(OthDg0}znc9|uTQcSjz~rsOe?B zqdhL+M-a)pZe|1_lwu_0vX3X}@@Wu0Ql49h`YACNAIXrDD^&4Jou&Bi&Q)$*9=4-< z8RhhK$(Li+(m;sXlsB^!3KxcJKOxwdTW}(*4x7I zpUoA&^AJhl&V8oMX4|^ez}ahCcvY=ZT8YXj{mt4VU`lDo>aqr|q6DwC6x?K_5CW$~ z)zg-9ANt?z62or1>nc!zuU+}sT#jur%jXAYhIHi7iyo)%A`$s%h>r5pNL+SskAUDd!qr3eDyxP-74e;H}tHGad%x`sHO*ag`H#Xky>3iFAj(Ov|A;N|? zKjF2rDFQY4o4r=Q|LF7ydQb(U7P$?x zjPEAHNHJG@?0Is^U8eK}WJTd&bf}X8kukM^WrL{o4oa&BLWR+$HA3c^qL2qPT2KSX z2b~gy718ts=bwuG;}6#ErogBbbuL-4u(B;0Ik`cg{ASMkM3MS&nT@?Q)9j`TEuOy)P`5lhQ1 z;%I{CMW9cWG)LZE&N3jhB4q8KClEZ&o@B@qHfl;eIde*KT`01pAf0UhaA*M@si zUV75{%4Da$zR5zw!(Vy)QBdb}tNq+(f0YTiV$ez#Ws(+sT59H@OB_X7zfxQ;yUK`d-ns(PF3 zbKMVlLcvG~nfcmFWc6)cDkV=Z?y^e`e*N1~bqI)x&CxxrmNSZ+a>af^fv(Vq;aZRe zZhPaXz~n!H$G0a^KV)VHRCwCRIP2ZfuQup)-*L3+C`F_yQRO=J86y4_4#U*&Fgc3j z?XKxB#(0IU)#J7=r`UG3r#vL~;_gpW?hYsDuZAKS5!WT>`8=*bmMQR=poBOpl`G0% z)G=|X#d}<{uD4f&l-Gf50mxVOTLI!78`USS0hz{z6=Y{btCf63x-~a?nyk|yzgX1X zBlzj*h({D1GC`!Yaa2MGScZfD9b)=Soq~M&2+`*c|6x{~D%VM1wYT6O*CqiJ93{gz zNKj(GP5#*fbR|GYmHp`5a%dGa{5?x#?B;sYLgUIE;vpsUl6!0 zG{$9mhp`ilcHOTPNW#iKetyCVG}=3LYuS+7k+6W5MXZ8n+`)`yetIWlOJHA;Fo^Q? zbGMSFN1&ItIBoW2@YDI<=1F*=BIRBoMjV44eg3=e1+o4)Z?EcVI5J{Uf;+CnNwcvV z==@09Dh8KYx1aEUH~dswv}+pr2stT`A4xmsO~&IJL5!WEH||fiHE-JdOT?7mc08ju z=029!D1#0og4NBRlt7&1cL>i#!wNKCs-Mgiedf-i_KehWfA z>tTu%eBc|R;haqf#?EI9e}_Y)*nFf)zBExSmn~nI#+2^4 zK#t@wOkNhzh9ta+Y_!H=xA(-NI{cW=S+gFe3x9J7S;wQ(bhQROO0vl~Ib}ZA_K^sEh6Jk3ffK0i}rSk($TX9OHDSY6B$m%$v#P??hE-Nzb z$^oglzM3^}qC+30F7M-;UzvQs^Y6$JughhvO&57+iq6cBNB_&Lw_x~fj%Xk}pNsr6 z?~-9fGkP9C(Q$Y;zywI?ie>;oDuP5@$->Hdg#-20)IGtC6pl{j6M%4L!bqNlW_rU- zfL1T5XHj?Dn5ta5 z_4{uDYhGgszU$gU{FmRsC_o`xoax{0egloVP2JVzqssLj6okL-kuNl213t3-g%+3_~P}g zOw+q}hu-nanqW~BIWoHT&1nYj!BwPT;$L}aPCd1jmFT8X9Tex}hvYRfInHwpVfs~8 z71`A%g_QzRN`6b~5TGbq`+YVHB$#fXr_vLN^D#;#svsPdZY^VX&3zk$#^(O?w8$}Oqdtdr>t#B4k9iJ zMh-7L(ZhIhPl9gJ1S{<@s~`L|AC@VzEI~}h4ZEnzKNmy4BrIqCHbc9lh%q$5NdIU- zo{+?lY=e+?YY8WrDDI2*csCuv$TFvOtYZ=<_gLcFE!J0cf|~kp3;t0iOb90vgx1}{z^k{F|^2e^)3$XdP+DXE3SlWc=w?yHhuHN34jER5142E`q zw^+>buAQ@oho-xXvMZ3?>dgu;i8Oo}5zUS6EnG`O-nuP>NkNOxO(yaGZVXX(%!VwF z-2Nqc&i!E1?HWH)=H=rj{Egx;>}lOZv!UMn~D{ z`fjt0G*{zMIX9L*c62W}(rLp`lIhJ=aZlCJ=uZ-b-(J?`wsqc%d({6q zu8HbQdNdN(M!7_~$SWtIAX&xo^e+!jk*$ymL-5UKs8HI}$)Zo|YXkewJ} z@&Xcko^+=16Jd*;k1r=Y$(S+Bsk}d783%hfAnR#3)PDx>LR0H{G*&BQ!;g~qQ$9__ z^WlzliHXv}*+KH(Y6GBv;`^!f{!OXP2IV+DrJqGxA5U>S8Cl<2+e(v$aK9MRMG$M# zZ$DJYbs(!LgQYt)Y?Xvt>W_pRaeKa7e=Zt|WEhS5uTlG_R_G4AWdNqRj%~84sN#w8 zY7%q3eiNsuYEK;vuGaNNL(PyNwiDfGZ^zi_j;+!3j%kYu$N; zt#u`7c3)0zdd~#GmH;dJSm%7Goy2UQ>LjKC`M&#O)%V%sI3VEGE(3mN@upA2lbirY zRwa$kzJK4G@(cU|A%u_BU7}{xM*1uXUTCPlC#`k zO4W7v`eoRrc(0p5oRas3{$j+?zybNW7c-h-R$cEf!>p5%9V9}^qyy;CHT?9$w*7@M zt?!>>@n3<8G+pi6-TVBqbwGYHT$w_K>bkWYA6x=vi~8e)NK7A#8CB9c6~{>kS4G+JBzueuSx2H8W2c0``tK;st`ermzM30~ z_6(h8nky(gXTiRk+z!;$Pc%rt*tTj;vq&ok0t_yyt7IFzbUH`Nl|BciEm{vOrq%IJ zhCQY(F8_D|ty!P9_ZvCs3;Z)lMOfkqZ)Zb22&03L%C+oDqqioM$mr?kvE$@a`C91o z|6%Wsc07%y1S*L$Q$t{R6`?t*T$0A`L#cD$T-8)!lvr}MycES8$`Gm0qJYxF#*An4 zmFgi{UZvRuYyw~H&Yvywz_$&+(s4j4wI8R&$It}2+}feN-0#6Ah4TTsJr4)|X{Xfr qifLzWTyFIME#?2x=1`GdQ2xpr3`ko|a0fQ3K>FIoT2% - {% block extra-headers %} {% endblock %} diff --git a/resources/templates/wiki.html b/resources/templates/wiki.html index 73cddbe..70d6974 100644 --- a/resources/templates/wiki.html +++ b/resources/templates/wiki.html @@ -1,5 +1,18 @@ {% extends "templates/base.html" %} +{% block extra-headers %} + + + + + +{% endblock %} + {% block content %}
{% if editable %} diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj new file mode 100644 index 0000000..9dd1d0c --- /dev/null +++ b/src/smeagol/formatting.clj @@ -0,0 +1,133 @@ +(ns ^{:doc "Format Semagol's enhanced markdown format." + :author "Simon Brooke"} + smeagol.formatting + (:require [clojure.string :as cs] + [cemerick.url :refer (url url-encode url-decode)] + [noir.io :as io] + [noir.session :as session] + [markdown.core :as md] + [smeagol.authenticate :as auth] + [clj-yaml.core :as yaml] + [clojure.data.json :as json])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: a very simple Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2017 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Right, doing the data visualisation thing is tricky. Doing it in the +;;;; pipeline doesn't work, because the md-to-html-string filter messes up +;;;; both YAML and JSON notation. So we need to extract the visualisation YAML +;;;; fragments from the Markdown text and replace them with tokens we will +;;;; recognise afterwards, perform md-to-html-string, and then replace our +;;;; tokens with the transformed visualisation specification. +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Error to show if text to be rendered is nil. +(def no-text-error "No text: does the file exist?") + + +(defn local-links + "Rewrite text in `html-src` surrounded by double square brackets as a local link into this wiki." + [^String html-src] + (if html-src + (cs/replace html-src #"\[\[[^\[\]]*\]\]" + #(let [text (clojure.string/replace %1 #"[\[\]]" "") + encoded (url-encode text) + ;; I use '\_' to represent '_' in wiki markup, because + ;; '_' is meaningful in Markdown. However, this needs to + ;; be stripped out when interpreting local links. + munged (cs/replace encoded #"%26%2395%3B" "_")] + (format "%s" munged text))) + no-text-error)) + + +(defn yaml->vis + "Transcode this YAML fragment into the source for a Vega visualisation with this index." + [^String yaml-src ^Integer index] + (str + "
\n" + "")) + + +(defn process-text + "Process this `text`, assumed to be markdown potentially containing both local links + and YAML visualisation specifications, and return a map comprising JSON visualisation + specification, and HTML text with markers for where those should be reinserted. + + The map has two top-level keys: `:visualisations`, a map of constructed keywords to + visualisation specifications, and `:text`, an HTML text string with the keywords + present where the corresponding visualisation should be inserted." + ([text] + (process-text 0 {:visualisations {}} (cs/split text #"```") '())) + ([index result fragments processed] + (cond + (empty? fragments) + (assoc result :text (local-links (md/md-to-html-string (cs/join "\n\n" (reverse processed))))) + (clojure.string/starts-with? (first fragments) "vis") + (let [kw (keyword (str "visualisation-" index))] + (process-text + (+ index 1) + (assoc + result + :visualisations + (assoc + (:visualisations result) + kw + (yaml->vis + (subs (first fragments) 3) + index))) + (rest fragments) + (cons kw processed))) + true + (process-text (+ index 1) result (rest fragments) (cons (first fragments) processed))))) + + +(defn reintegrate-visualisations + "Given a map of the form produced by `process-text`, return a string of HTML text + with the visualisations (if any) reintegrated." + ([processed-text] + (reintegrate-visualisations (:visualisations processed-text) (:text processed-text))) + ([visualisations text] + (let [ks (keys visualisations)] + (if (empty? (keys visualisations)) + text + (let [kw (first ks)] + (reintegrate-visualisations + (dissoc visualisations kw) + (cs/replace + text + (str kw) + (cs/replace (kw visualisations) "\\/" "/")))))))) + + +(defn md->html + "Take this markdown source, and return HTML." + [md-src] + (reintegrate-visualisations (process-text md-src))) + + diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index 52f7aba..85f6dd9 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -14,6 +14,7 @@ [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] [smeagol.diff2html :as d2h] + [smeagol.formatting :refer [md->html]] [smeagol.layout :as layout] [smeagol.util :as util] [smeagol.history :as hist] @@ -98,7 +99,7 @@ (merge (util/standard-params request) {:title (str (:edit-title-prefix layout/config) " " page) :page page - :side-bar (util/local-links (util/md->html side-bar)) + :side-bar (md->html (io/slurp-resource side-bar)) :content (if exists? (io/slurp-resource (str "/content/" page suffix)) "") :exists exists?})))))) @@ -124,8 +125,7 @@ (merge (util/standard-params request) {:title page :page page - :side-bar (util/local-links (util/md->html "/content/_side-bar.md")) - :content (util/local-links (util/md->html file-name)) + :content (md->html (io/slurp-resource file-name)) :editable true}))) true (response/redirect (str "/edit?page=" page))))) @@ -138,11 +138,12 @@ page (url-decode (or (:page params) (:default-page-title layout/config))) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] + (timbre/info (format "Showing history of page '%s'" page)) (layout/render "history.html" (merge (util/standard-params request) {:title (str "History of " page) :page page - :history (hist/find-history repo-path file-name)})))) + :history (md->html (hist/find-history repo-path file-name))})))) (defn version-page @@ -152,15 +153,14 @@ page (url-decode (or (:page params) (:default-page-title layout/config))) version (:version params) file-name (str page ".md") - repo-path (str (io/resource-path) "/content/")] + repo-path (str (io/resource-path) "/content/") + content (hist/fetch-version repo-path file-name version)] + (timbre/info (format "Showing version '%s' of page '%s'" version page)) (layout/render "wiki.html" (merge (util/standard-params request) {:title (str (:vers-col-hdr layout/config) " " version " of " page) :page page - :content (util/local-links - (md/md-to-html-string - (hist/fetch-version - repo-path file-name version)))})))) + :content (md->html content)})))) (defn diff-page @@ -171,6 +171,7 @@ version (:version params) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] + (timbre/info (format "Showing diff between version '%s' of page '%s' and current" version page)) (layout/render "wiki.html" (merge (util/standard-params request) {:title (str (:diff-title-prefix layout/config)" " version " of " page) @@ -201,10 +202,7 @@ (layout/render "auth.html" (merge (util/standard-params request) {:title (if user (str (:logout-link layout/config) " " user) (:login-link layout/config)) - :redirect-to ((:headers request) "referer") - :side-bar (util/local-links (util/md->html "/content/_side-bar.md")) - :header (util/local-links (util/md->html "/content/_header.md")) - :user user}))))) + :redirect-to ((:headers request) "referer")}))))) (defn passwd-page @@ -221,8 +219,6 @@ (layout/render "passwd.html" (merge (util/standard-params request) {:title (str (:chpass-title-prefix layout/config) " " user) - :side-bar (util/local-links (util/md->html "/content/_side-bar.md")) - :header (util/local-links (util/md->html "/content/_header.md")) :message (if changed? (:chpass-success layout/config)) :error (cond (nil? oldpass) nil diff --git a/src/smeagol/util.clj b/src/smeagol/util.clj index 67f7203..d0c2131 100644 --- a/src/smeagol/util.clj +++ b/src/smeagol/util.clj @@ -1,12 +1,9 @@ (ns ^{:doc "Miscellaneous utility functions supporting Smeagol." :author "Simon Brooke"} smeagol.util - (:require [clojure.string :as cs] - [cemerick.url :refer (url url-encode url-decode)] - [noir.io :as io] - [noir.session :as session] - [markdown.core :as md] - [smeagol.authenticate :as auth])) + (:require [noir.session :as session] + [smeagol.authenticate :as auth] + [smeagol.formatting :refer [md->html]])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; @@ -31,38 +28,13 @@ ;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(defn md->html - "reads a markdown file from public/md and returns an HTML string" - [filename] - (md/md-to-html-string (io/slurp-resource filename))) - - -;; Error to show if text to be rendered is nil. -(def no-text-error "No text: does the file exist?") - - -(defn local-links - "Rewrite text in `html-src` surrounded by double square brackets as a local link into this wiki." - [^String html-src] - (if html-src - (cs/replace html-src #"\[\[[^\[\]]*\]\]" - #(let [text (clojure.string/replace %1 #"[\[\]]" "") - encoded (url-encode text) - ;; I use '\_' to represent '_' in wiki markup, because - ;; '_' is meaningful in Markdown. However, this needs to - ;; be stripped out when interpreting local links. - munged (cs/replace encoded #"%26%2395%3B" "_")] - (format "%s" munged text))) - no-text-error)) - - (defn standard-params "Return a map of standard parameters to pass to the template renderer." [request] (let [user (session/get :user)] {:user user :admin (auth/get-admin user) - :side-bar (local-links (md->html "/content/_side-bar.md")) - :header (local-links (md->html "/content/_header.md")) + :side-bar (md->html (io/slurp-resource "/content/_side-bar.md")) + :header (md->html (io/slurp-resource "/content/_header.md")) :version (System/getProperty "smeagol.version")})) From c2e771561d583b61767c1c4879a251f1e0f836fe Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 26 Jul 2017 16:51:31 +0100 Subject: [PATCH 04/14] #17: Fix. --- resources/config.edn | 4 +++ resources/templates/base.html | 3 +- resources/templates/upload.html | 34 ++++++++++++++++++ src/smeagol/routes/wiki.clj | 26 ++++++++++++++ src/smeagol/uploads.clj | 64 +++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 resources/templates/upload.html create mode 100644 src/smeagol/uploads.clj diff --git a/resources/config.edn b/resources/config.edn index 6afb713..3930d1c 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -65,6 +65,10 @@ :edit-users-title "Select user to edit" ;; title of edit users page :email-prompt "Email address" ;; text of the email widget prompt on edit user page + :file-upload-link-text "You may link to this file using a link of the form" + ;; Text introducing the link to an uploaded file + :file-upload-prompt "File to upload" ;; prompt string for the file upload widget + :file-upload-title "Upload a file" ;; title for the file upload page :is-admin-prompt "Is administrator?" :home-link "Home" ;; text of the home link on the menu :login-label "Log in!" ;; text of the login widget on the login page diff --git a/resources/templates/base.html b/resources/templates/base.html index 754b31d..8d980d8 100644 --- a/resources/templates/base.html +++ b/resources/templates/base.html @@ -6,6 +6,7 @@ + {% block extra-headers %} {% endblock %} @@ -20,6 +21,7 @@
  • {{config.edit-users-link}}
    • {% endif %} {% if user %} +
  • {{config.file-upload-title}}
  • {{config.change-pass-link}}
  • {{config.logged-in-as}} {{user}}
  • @@ -90,4 +92,3 @@
    - diff --git a/resources/templates/upload.html b/resources/templates/upload.html new file mode 100644 index 0000000..ece03f4 --- /dev/null +++ b/resources/templates/upload.html @@ -0,0 +1,34 @@ +{% extends "templates/base.html" %} +{% block content %} + Uploaded: {{uploaded}}; is-image: {{is-image}} +
    + {% if uploaded %} + {% if is-image %} + Uploaded image + +

    + {{config.file-upload-link-text}}: + + ![Uploaded image](uploads/{{uploaded}}) +

    + {% else %} +

    + {{config.file-upload-link-text}}: + + [Uploaded file](uploads/{{uploaded}}) +

    + {% endif %} + {% else %} +
    +

    + + +

    +

    + + +

    +
    + {% endif %} +
    +{% endblock %} diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index 85f6dd9..e67b6e9 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -3,6 +3,7 @@ smeagol.routes.wiki (:require [clojure.walk :refer :all] [clojure.java.io :as cjio] + [clojure.string :as cs] [cemerick.url :refer (url url-encode url-decode)] [compojure.core :refer :all] [clj-jgit.porcelain :as git] @@ -17,6 +18,7 @@ [smeagol.formatting :refer [md->html]] [smeagol.layout :as layout] [smeagol.util :as util] + [smeagol.uploads :as ul] [smeagol.history :as hist] [smeagol.routes.admin :as admin])) @@ -145,6 +147,28 @@ :page page :history (md->html (hist/find-history repo-path file-name))})))) +(defn upload-page + "Render a form to allow the upload of a file." + [request] + (let [params (keywordize-keys (:params request)) + data-path (str (io/resource-path) "/uploads/") + upload (:upload params) + uploaded (if upload (ul/store-upload params))] + (layout/render "upload.html" + (merge (util/standard-params request) + {:title (:file-upload-title layout/config) + :uploaded uploaded + :is-image (and + uploaded + (or + (cs/ends-with? uploaded ".gif") + (cs/ends-with? uploaded ".jpg") + (cs/ends-with? uploaded ".jpeg") + (cs/ends-with? uploaded ".png") + (cs/ends-with? uploaded ".GIF") + (cs/ends-with? uploaded ".JPG") + (cs/ends-with? uploaded ".PNG")))})))) + (defn version-page "Render a specific historical version of a page" @@ -246,4 +270,6 @@ (POST "/auth" request (auth-page request)) (GET "/passwd" request (passwd-page request)) (POST "/passwd" request (passwd-page request)) + (GET "/upload" request (upload-page request)) + (POST "/upload" request (upload-page request)) ) diff --git a/src/smeagol/uploads.clj b/src/smeagol/uploads.clj new file mode 100644 index 0000000..dbc31af --- /dev/null +++ b/src/smeagol/uploads.clj @@ -0,0 +1,64 @@ +(ns ^{:doc "Handle file uploads." + :author "Simon Brooke"} + smeagol.uploads + (:import [java.io File]) + (:require [clojure.string :as cs] + [noir.io :as io] + [taoensso.timbre :as timbre])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: a very simple Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2017 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn avoid-name-collisions + "Find a filename within this `path`, based on this `file-name`, that does not + reference an existing file. It is assumed that `path` ends with a path separator. + Returns a filename hwich does not currently reference a file within the path." + [path file-name] + (if (.exists (File. (str path file-name))) + (let [parts (cs/split file-name #"\.") + prefix (cs/join "." (butlast parts)) + suffix (last parts)] + (first + (filter #(not (.exists (File. (str path %)))) + (map #(str prefix "." % "." suffix) (range))))) + file-name)) + + +(defn store-upload + "Store an upload both to the file system and to the database. + The issue with storing an upload is moving it into place. + If `params` are passed as a map, it is expected that this is a map from + an HTTP POST operation of a form with type `multipart/form-data`." + [params] + (let [upload (:upload params) + tmp-file (:tempfile upload) + path (str (io/resource-path) "uploads/") + filename (avoid-name-collisions path (:filename upload))] + (timbre/info + (str "Storing upload file: " upload)) + (if tmp-file + (do + (.renameTo tmp-file + (File. (str path filename))) + filename) + (throw (Exception. "No file found?"))))) From c289f10524832de9d79e5a7d667d61cc07f31b98 Mon Sep 17 00:00:00 2001 From: simon Date: Wed, 26 Jul 2017 17:07:40 +0100 Subject: [PATCH 05/14] Updated introduction and README to say you can now upload files. --- README.md | 4 +++- resources/public/content/Introduction.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 23aa57c..068a5f3 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Security is now greatly improved. There is a file called *passwd* in the *resour that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. ## Images -Smeagol does not currently have any mechanism to upload images. You can, however, link to images already available on the web, like this: +You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: ![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) @@ -51,6 +51,8 @@ encoding: aggregate: sum ``` +Data files can be uploaded in the same way as images, by using the **upload a file** link. + Note that this visualisation will not be rendered in the GitHub wiki, as it doesn't have Smeagol's data visualisation magic. This is what it should look like: ![Example visualisation](https://github.com/simon-brooke/smeagol/blob/develop/resources/public/data/london.png?raw=true) diff --git a/resources/public/content/Introduction.md b/resources/public/content/Introduction.md index 23aa57c..068a5f3 100644 --- a/resources/public/content/Introduction.md +++ b/resources/public/content/Introduction.md @@ -20,7 +20,7 @@ Security is now greatly improved. There is a file called *passwd* in the *resour that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. ## Images -Smeagol does not currently have any mechanism to upload images. You can, however, link to images already available on the web, like this: +You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: ![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) @@ -51,6 +51,8 @@ encoding: aggregate: sum ``` +Data files can be uploaded in the same way as images, by using the **upload a file** link. + Note that this visualisation will not be rendered in the GitHub wiki, as it doesn't have Smeagol's data visualisation magic. This is what it should look like: ![Example visualisation](https://github.com/simon-brooke/smeagol/blob/develop/resources/public/data/london.png?raw=true) From fd5cd110243a5aaf6e3586b9fc8841f53787c495 Mon Sep 17 00:00:00 2001 From: simon Date: Sun, 30 Jul 2017 14:40:56 +0100 Subject: [PATCH 06/14] Experimental tidying up after vega introduction None of this is very significant, but it is small improvements. --- project.clj | 5 ++++- resources/templates/base.html | 3 +-- resources/templates/edit.html | 4 ++-- resources/templates/wiki.html | 4 +++- src/smeagol/formatting.clj | 6 +++++- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/project.clj b/project.clj index 347c7ed..dba11fb 100644 --- a/project.clj +++ b/project.clj @@ -33,7 +33,10 @@ [lein-bower "0.5.1"] [lein-ancient "0.5.5" :exclusions [org.clojure/clojure org.clojure/data.xml]] [lein-marginalia "0.7.1" :exclusions [org.clojure/clojure]]] - :bower-dependencies [[simplemde "1.11.2"]] + :bower-dependencies [[simplemde "1.11.2"] + ;; [vega-embed "3.0.0-beta.19"] vega-embed currently not loaded from Bower because of + ;; dependency conflict which will hopefully be resolved soon. + [vega-lite "2.0.0-beta.10"]] :ring {:handler smeagol.handler/app :init smeagol.handler/init :destroy smeagol.handler/destroy} diff --git a/resources/templates/base.html b/resources/templates/base.html index 8d980d8..103a537 100644 --- a/resources/templates/base.html +++ b/resources/templates/base.html @@ -5,8 +5,7 @@ - - + {% style "/content/stylesheet.css" %} {% block extra-headers %} {% endblock %} diff --git a/resources/templates/edit.html b/resources/templates/edit.html index 8f32002..a5672a7 100644 --- a/resources/templates/edit.html +++ b/resources/templates/edit.html @@ -1,7 +1,7 @@ {% extends "templates/base.html" %} {% block extra-headers %} - - +{% style "/vendor/simplemde/dist/simplemde.min.css" %} +{% script "/vendor/simplemde/dist/simplemde.min.js" %} {% endblock %} {% block content %} diff --git a/resources/templates/wiki.html b/resources/templates/wiki.html index 70d6974..97d0228 100644 --- a/resources/templates/wiki.html +++ b/resources/templates/wiki.html @@ -2,7 +2,9 @@ {% block extra-headers %} - + + {% script "/vendor/vega-lite/build/vega-lite.js" %} + + + {% style "vendor/mermaid/dist/mermaid.css" %} + {% script "vendor/mermaid/dist/mermaid.js" %} {% endblock %} {% block content %} @@ -25,4 +31,14 @@ package for vega-embed, so we're currently not installing either it or Vega loca {% endif %} {{content|safe}}
    • + + {% endblock %} diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index 8acd922..e245749 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -73,6 +73,12 @@ (assoc (yaml/parse-string yaml-src) (keyword "$schema") "https://vega.github.io/schema/vega-lite/v2.json")) ";\nvega.embed('#vis" index "', vl" index ");\n//]]\n")) +(defn process-mermaid + "Lightly mung the mermaid specification." + [^String graph-spec ^Integer index] + (str "
    \n" + graph-spec + "\n
    ")) (defn process-text "Process this `text`, assumed to be markdown potentially containing both local links @@ -92,6 +98,7 @@ (md/md-to-html-string (cs/join "\n\n" (reverse processed)) :heading-anchors true))) + ;;; TODO: refactor; generalise extension architecture (clojure.string/starts-with? (first fragments) "vis") (let [kw (keyword (str "visualisation-" index))] (process-text @@ -107,6 +114,21 @@ index))) (rest fragments) (cons kw processed))) + (clojure.string/starts-with? (first fragments) "mermaid") + (let [kw (keyword (str "visualisation-" index))] + (process-text + (+ index 1) + (assoc + result + :visualisations + (assoc + (:visualisations result) + kw + (process-mermaid + (subs (first fragments) 7) + index))) + (rest fragments) + (cons kw processed))) true (process-text (+ index 1) result (rest fragments) (cons (first fragments) processed))))) diff --git a/test/smeagol/test/util.clj b/test/smeagol/test/util.clj index d5e94f9..b1f21f8 100644 --- a/test/smeagol/test/util.clj +++ b/test/smeagol/test/util.clj @@ -1,7 +1,7 @@ (ns smeagol.test.util (:use clojure.test ring.mock.request - smeagol.util)) + smeagol.formatting)) (deftest test-local-links (testing "Rewriting of local links" From 91367e40aae7a1b68ab6bf0af287e7fe4e6662e2 Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Wed, 2 Aug 2017 07:36:37 +0100 Subject: [PATCH 08/14] More orthogonal inclusion processing --- src/smeagol/formatting.clj | 62 +++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index e245749..c9323c6 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -61,35 +61,43 @@ no-text-error)) -(defn yaml->vis - "Transcode this YAML fragment into the source for a Vega visualisation with this index." - [^String yaml-src ^Integer index] +(defn yaml->json + "Rewrite this string, assumed to be in YAML format, as JSON." + [^String yaml-src] + (json/write-str (yaml/parse-string yaml-src))) + + +(defn process-vega + "Process this `vega-source` string, assumed to be in YAML format, into a specification + of a Vega chart, and add the plumbing to render it." + [^String vega-src ^Integer index] (str "
    \n" "")) + (defn process-mermaid - "Lightly mung the mermaid specification." + "Lightly mung this `graph-spec`, assumed to be a mermaid specification." [^String graph-spec ^Integer index] (str "
    \n" graph-spec "\n
    ")) + (defn process-text "Process this `text`, assumed to be markdown potentially containing both local links and YAML visualisation specifications, and return a map comprising JSON visualisation specification, and HTML text with markers for where those should be reinserted. - The map has two top-level keys: `:visualisations`, a map of constructed keywords to - visualisation specifications, and `:text`, an HTML text string with the keywords - present where the corresponding visualisation should be inserted." + The map has two top-level keys: `:inclusions`, a map of constructed keywords to + inclusion specifications, and `:text`, an HTML text string with the keywords + present where the corresponding inclusion should be inserted." ([text] - (process-text 0 {:visualisations {}} (cs/split text #"```") '())) + (process-text 0 {:inclusions {}} (cs/split text #"```") '())) ([index result fragments processed] (cond (empty? fragments) @@ -100,29 +108,29 @@ :heading-anchors true))) ;;; TODO: refactor; generalise extension architecture (clojure.string/starts-with? (first fragments) "vis") - (let [kw (keyword (str "visualisation-" index))] + (let [kw (keyword (str "inclusion-" index))] (process-text (+ index 1) (assoc result - :visualisations + :inclusions (assoc - (:visualisations result) + (:inclusions result) kw - (yaml->vis + (process-vega (subs (first fragments) 3) index))) (rest fragments) (cons kw processed))) (clojure.string/starts-with? (first fragments) "mermaid") - (let [kw (keyword (str "visualisation-" index))] + (let [kw (keyword (str "inclusion-" index))] (process-text (+ index 1) (assoc result - :visualisations + :inclusions (assoc - (:visualisations result) + (:inclusions result) kw (process-mermaid (subs (first fragments) 7) @@ -133,27 +141,27 @@ (process-text (+ index 1) result (rest fragments) (cons (first fragments) processed))))) -(defn reintegrate-visualisations +(defn reintegrate-inclusions "Given a map of the form produced by `process-text`, return a string of HTML text - with the visualisations (if any) reintegrated." + with the inclusions (if any) reintegrated." ([processed-text] - (reintegrate-visualisations (:visualisations processed-text) (:text processed-text))) - ([visualisations text] - (let [ks (keys visualisations)] - (if (empty? (keys visualisations)) + (reintegrate-inclusions (:inclusions processed-text) (:text processed-text))) + ([inclusions text] + (let [ks (keys inclusions)] + (if (empty? (keys inclusions)) text (let [kw (first ks)] - (reintegrate-visualisations - (dissoc visualisations kw) + (reintegrate-inclusions + (dissoc inclusions kw) (cs/replace text (str kw) - (cs/replace (kw visualisations) "\\/" "/")))))))) + (cs/replace (kw inclusions) "\\/" "/")))))))) (defn md->html "Take this markdown source, and return HTML." [md-src] - (reintegrate-visualisations (process-text md-src))) + (reintegrate-inclusions (process-text md-src))) From 19a715c4fb7029023f95a35c6ccba95f9f8c2c76 Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Mon, 7 Aug 2017 23:21:45 +0100 Subject: [PATCH 09/14] A very long way towards extensible formatters --- src/smeagol/formatting.clj | 112 ++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 40 deletions(-) diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index c9323c6..5ed74a8 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -6,6 +6,7 @@ [noir.io :as io] [noir.session :as session] [markdown.core :as md] + [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] [clj-yaml.core :as yaml] [clojure.data.json :as json])) @@ -35,7 +36,7 @@ ;;;; ;;;; Right, doing the data visualisation thing is tricky. Doing it in the ;;;; pipeline doesn't work, because the md-to-html-string filter messes up -;;;; both YAML and JSON notation. So we need to extract the visualisation YAML +;;;; both YAML and JSON notation. So we need to extract the visualisation ;;;; fragments from the Markdown text and replace them with tokens we will ;;;; recognise afterwards, perform md-to-html-string, and then replace our ;;;; tokens with the transformed visualisation specification. @@ -88,6 +89,63 @@ "\n
    ")) +;; TODO: This isn't (yet) exactly what I want. The formatters ought to be configurable +;; without editing the Smeagol code directly. But it's a long way in the right direction. +(def ^:dynamic *formatters* + {"vega" process-vega + "vis" process-vega + "mermaid" process-mermaid}) + + +(defn get-first-token + "Return the first space-separated token of this `string`." + [^String string] + (if string (first (cs/split string #"[^a-zA-Z0-9]+")))) + + +(defn- process-markdown-fragment + "Within the context of `process-text`, process a fragment believed to be markdown. + + As with `process-text`, this function returns a map with two top-level keys: + `:inclusions`, a map of constructed keywords to inclusion specifications, + and `:text`, an HTML text string with the keywords present where the + corresponding inclusion should be inserted." + [index result fragments processed] + (process-text + (+ index 1) + result + (rest fragments) + (cons (first fragments) processed))) + + +(defn- apply-formatter + "Within the context of `process-text`, process a fragment for which an explicit + §formatter has been identified. + + As with `process-text`, this function returns a map with two top-level keys: + `:inclusions`, a map of constructed keywords to inclusion specifications, + and `:text`, an HTML text string with the keywords present where the + corresponding inclusion should be inserted." + [index result fragments processed fragment token formatter] + (let + [kw (keyword (str "inclusion-" index))] + (process-text + (+ index 1) + (assoc + result + :inclusions + (assoc + (:inclusions result) + kw + (apply + formatter + (list + (subs fragment (count token)) + index)))) + (rest fragments) + (cons kw processed)))) + + (defn process-text "Process this `text`, assumed to be markdown potentially containing both local links and YAML visualisation specifications, and return a map comprising JSON visualisation @@ -99,46 +157,20 @@ ([text] (process-text 0 {:inclusions {}} (cs/split text #"```") '())) ([index result fragments processed] - (cond - (empty? fragments) - (assoc result :text - (local-links + (let [fragment (first fragments) + first-token (get-first-token fragment) + formatter (*formatters* first-token)] + (cond + (empty? fragments) + (assoc result :text + (local-links (md/md-to-html-string - (cs/join "\n\n" (reverse processed)) - :heading-anchors true))) - ;;; TODO: refactor; generalise extension architecture - (clojure.string/starts-with? (first fragments) "vis") - (let [kw (keyword (str "inclusion-" index))] - (process-text - (+ index 1) - (assoc - result - :inclusions - (assoc - (:inclusions result) - kw - (process-vega - (subs (first fragments) 3) - index))) - (rest fragments) - (cons kw processed))) - (clojure.string/starts-with? (first fragments) "mermaid") - (let [kw (keyword (str "inclusion-" index))] - (process-text - (+ index 1) - (assoc - result - :inclusions - (assoc - (:inclusions result) - kw - (process-mermaid - (subs (first fragments) 7) - index))) - (rest fragments) - (cons kw processed))) - true - (process-text (+ index 1) result (rest fragments) (cons (first fragments) processed))))) + (cs/join "\n\n" (reverse processed)) + :heading-anchors true))) + formatter + (apply-formatter index result fragments processed fragment first-token formatter) + true + (process-markdown-fragment index result fragments processed))))) (defn reintegrate-inclusions From a2b6d6f72b0051b485977be1631dbc819c499ad1 Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Tue, 8 Aug 2017 09:01:35 +0100 Subject: [PATCH 10/14] Initial work on i18n --- resources/i18n/en-GB.edn | 108 +++++++++++++++++++++++++++++++++++ src/smeagol/formatting.clj | 3 + src/smeagol/i18n.clj | 53 +++++++++++++++++ src/smeagol/layout.clj | 9 +-- src/smeagol/routes/admin.clj | 14 ++--- src/smeagol/routes/wiki.clj | 35 ++++++------ src/smeagol/util.clj | 8 +++ 7 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 resources/i18n/en-GB.edn create mode 100644 src/smeagol/i18n.clj diff --git a/resources/i18n/en-GB.edn b/resources/i18n/en-GB.edn new file mode 100644 index 0000000..a6de517 --- /dev/null +++ b/resources/i18n/en-GB.edn @@ -0,0 +1,108 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: a very simple Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2017 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;;; en-GB.edn: English-language messages. +;;; This is essentially all the text in the chrome - that which isn't editable +;;; through the wiki itself + +;; ; ; ; ; ; ; ; ; ; +{:add-user-label "Add new user" ;; label for the add user link on edit users page + :change-pass-label "Change password!" + ;; text of the change password widget itself on the + ;; change password page + :change-pass-link "Change password" + ;; text of the change password link on the menu + :change-pass-prompt "To change your password" + ;; text of the change password widget prompt on the + ;; change password page + :change-col-hdr "Changes" ;; header for the changes column in history + :chpass-bad-match "Your proposed passwords don't match" + ;; error text if proposed passwords don't match + :chpass-fail "Your password was not changed" + ;; error text on fail other htan too short or bad match + :chpass-success "Your password was changed" + ;; confirmation text on password change + :chpass-too-short "You proposed password wasn't long enough: eight characters required" + ;; error text if proposed password is too short + :chpass-title-prefix "Change password for" + ;; prefix for title of change password page + :cookies-about "About cookies" ;; about cookies text + :cookies-more "This website stores session information as a 'cookie' on your browser. This helps us show you the content you want to see. This cookie does not identify you, and cannot be read by other websites. It is deleted by your browser as soon as you leave this site. This website does not use any third party cookies, so your visit here cannot be tracked by other websites." + ;; more about cookies text + :default-page-title "Introduction" ;; title of the default page in this wiki + :del-col-hdr "Delete" ;; header for delete column on edit users page + :del-user-fail "Could not delete user" + ;; error message on failure to delete user + :del-user-success "Successfully deleted user" + ;; confirmation message on deletion of user + :diff-title-prefix "Changes since version" + ;; prefix for the header of the changes page + :edit-col-hdr "Edit" ;; header for edit column on edit users page + :edit-page-link "Edit this page" + ;; text of the edit page link on the content frame + :edit-title-prefix "Edit" ;; prefix for title of edit content page + :edit-users-link "Edit users" ;; text of the edit users link on the menu + :edit-users-title "Select user to edit" + ;; title of edit users page + :email-prompt "Email address" ;; text of the email widget prompt on edit user page + :file-upload-link-text "You may link to this file using a link of the form" + ;; Text introducing the link to an uploaded file + :file-upload-prompt "File to upload" ;; prompt string for the file upload widget + :file-upload-title "Upload a file" ;; title for the file upload page + :is-admin-prompt "Is administrator?" + :home-link "Home" ;; text of the home link on the menu + :login-label "Log in!" ;; text of the login widget on the login page + :login-link "Log in" ;; text of the login link on the menu + :login-prompt "To edit this wiki" + ;; text of the action widget prompt on the login page + :logout-label "Log out!" ;; text of the logout widget on the logout page + :logout-link "Log out" ;; text of the logout link on the menu + :logged-in-as "You are logged in as" + ;; text of the 'logged in as' label on the menu + :history-link "History" ;; text of the history link on the content frame + :history-title-prefix "History of" ;; prefix of the title on the history page + :new-pass-prompt "New password" ;; text of the new password widget prompt on the change + ;; password and edit user pages + :old-pass-prompt "Your password" + ;; text of the old password widget prompt on the change + ;; password page, and password widget on login page + :rpt-pass-prompt "And again" ;; text of the new password widget prompt on the change + ;; password and edit user pages + :save-prompt "When you have finished editing" + ;; text of the save widget label on edit content + ;; and edit user page + :save-label "Save!" ;; text of the save widget itself + :save-user-fail "Failed to store user" + :save-user-success "Successfully stored user" + :site-title "Smeagol" ;; overall title of the site, used in page headings + :username-prompt "Username" ;; text of the username widget prompt on edit user page + ;; text of the is admin widget prompt on edit user page + :user-title-prefix "Edit user" ;; prefix for title of edit user page + :vers-col-hdr "Version" ;; header for the version column in history + :what-col-hdr "What" ;; header for the what column in history + :what-changed-prompt "What have you changed?" + ;; text of the summary widget prompt on edit + ;; content page + :when-col-hdr "When" ;; header for the when column in history + :your-uname-prompt "Your username" ;; text of the username widget prompt on the login page + } diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index 5ed74a8..daa5f66 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -68,6 +68,9 @@ (json/write-str (yaml/parse-string yaml-src))) +(declare process-text) + + (defn process-vega "Process this `vega-source` string, assumed to be in YAML format, into a specification of a Vega chart, and add the plumbing to render it." diff --git a/src/smeagol/i18n.clj b/src/smeagol/i18n.clj new file mode 100644 index 0000000..2e259a2 --- /dev/null +++ b/src/smeagol/i18n.clj @@ -0,0 +1,53 @@ +(ns ^{:doc "Internationalisation." + :author "Simon Brooke"} + smeagol.i18n + (:require [noir.session :as session] + [noir.io :as io] + [smeagol.authenticate :as auth] + [smeagol.formatting :refer [md->html]])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: a very simple Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2014 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(def i18n-cache {}) + + +(defn parse-accept-language + "The `Accept-Language` header is defined in section 14.4 of RFC 2616. It is composed + of a sequence of 'language tags' defined in section 3.10 of the same document. + Generally a language tag is a short alpha string (the 'primary language tag'), + optionally followed by a hyphen and another short alpha string (a 'sub tag'). + A sequence of more than one sub-tag is allowed. + + Generally a two-character primary tag will be an ISO-639 language code and any + initial two-character sub tag will be an ISO-3166 country code. + + Each language tag may optionally followed by a semi-colon followed by a 'q' value, + specified `q=0.8` where the numeric value is a real number in the range 0...1 + and represents the user's preference for this language (higher is better). If + no q value is supplied 1 is assumed. + + Language specifiers are separated by a comma followed by a space." + [accept-language-header] + ) diff --git a/src/smeagol/layout.clj b/src/smeagol/layout.clj index 547b126..20cc737 100644 --- a/src/smeagol/layout.clj +++ b/src/smeagol/layout.clj @@ -7,7 +7,8 @@ [noir.io :as io] [ring.util.response :refer [content-type response]] [compojure.response :refer [Renderable]] - [environ.core :refer [env]])) + [environ.core :refer [env]] + [smeagol.util :as util])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; @@ -34,10 +35,6 @@ (def template-path "templates/") -;; the relative path to the config file. -(def config-file-path (str (io/resource-path) "../config.edn")) - -(def config (read-string (slurp config-file-path))) (deftype RenderableTemplate [template params] Renderable @@ -45,7 +42,7 @@ (content-type (->> (assoc params (keyword (s/replace template #".html" "-selected")) "active" - :config config + :config util/config :dev (env :dev) :servlet-context (if-let [context (:servlet-context request)] diff --git a/src/smeagol/routes/admin.clj b/src/smeagol/routes/admin.clj index 184335d..64dc239 100644 --- a/src/smeagol/routes/admin.clj +++ b/src/smeagol/routes/admin.clj @@ -39,7 +39,7 @@ user (session/get :user)] (layout/render "edit-users.html" (merge (util/standard-params request) - {:title (:edit-users-title layout/config) + {:title (:edit-users-title util/config) :users (auth/list-users)})))) (defn delete-user @@ -48,11 +48,11 @@ (let [params (keywordize-keys (:params request)) target (:target params) deleted (auth/delete-user target) - message (if deleted (str (:del-user-success layout/config) " " target ".")) - error (if (not deleted) (str (:del-user-fail layout/config) " " target "."))] + message (if deleted (str (:del-user-success util/config) " " target ".")) + error (if (not deleted) (str (:del-user-fail util/config) " " target "."))] (layout/render "edit-users.html" (merge (util/standard-params request) - {:title (:edit-users-title layout/config) + {:title (:edit-users-title util/config) :message message :error error :users (auth/list-users)})))) @@ -67,9 +67,9 @@ password (if (and pass1 (auth/evaluate-password pass1 (:pass2 params))) pass1) stored (if (:email params) (auth/add-user target password (:email params) (:admin params))) - message (if stored (str (:save-user-success layout/config) " " target ".")) + message (if stored (str (:save-user-success util/config) " " target ".")) error (if (and (:email params) (not stored)) - (str (:save-user-fail layout/config) " " target ".")) + (str (:save-user-fail util/config) " " target ".")) details (auth/fetch-user-details target)] (if message (timbre/info message)) @@ -77,7 +77,7 @@ (timbre/warn error)) (layout/render "edit-user.html" (merge (util/standard-params request) - {:title (str (:edit-title-prefix layout/config) " " target) + {:title (str (:edit-title-prefix util/config) " " target) :message message :error error :target target diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index e6ccccb..2a0fd0b 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -77,14 +77,14 @@ (if (= suffix ".md") (url-encode page) - (:default-page-title layout/config)))))) + (:default-page-title util/config)))))) (defn edit-page "Render a page in a text-area for editing. This could have been done in the same function as wiki-page, and that would have been neat, but I couldn't see how to establish security if that were done." ([request] - (edit-page request (:default-page-title layout/config) ".md" "edit.html" "/content/_edit-side-bar.md")) + (edit-page request (:default-page-title util/config) ".md" "edit.html" "/content/_edit-side-bar.md")) ([request default suffix template side-bar] (let [params (keywordize-keys (:params request)) src-text (:src params) @@ -99,7 +99,7 @@ true (layout/render template (merge (util/standard-params request) - {:title (str (:edit-title-prefix layout/config) " " page) + {:title (str (:edit-title-prefix util/config) " " page) :page page :side-bar (md->html (io/slurp-resource side-bar)) :content (if exists? (io/slurp-resource (str "/content/" page suffix)) "") @@ -115,8 +115,9 @@ (defn wiki-page "Render the markdown page specified in this `request`, if any. If none found, redirect to edit-page" [request] + (timbre/debug (str "Request map: " request)) (let [params (keywordize-keys (:params request)) - page (or (:page params) (:default-page-title layout/config)) + page (or (:page params) (:default-page-title util/config)) file-name (str "/content/" page ".md") file-path (str (io/resource-path) file-name) exists? (.exists (clojure.java.io/as-file file-path))] @@ -137,7 +138,7 @@ if any. If none, error?" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title layout/config))) + page (url-decode (or (:page params) (:default-page-title util/config))) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] (timbre/info (format "Showing history of page '%s'" page)) @@ -156,7 +157,7 @@ uploaded (if upload (ul/store-upload params))] (layout/render "upload.html" (merge (util/standard-params request) - {:title (:file-upload-title layout/config) + {:title (:file-upload-title util/config) :uploaded uploaded :is-image (and uploaded @@ -174,7 +175,7 @@ "Render a specific historical version of a page" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title layout/config))) + page (url-decode (or (:page params) (:default-page-title util/config))) version (:version params) file-name (str page ".md") repo-path (str (io/resource-path) "/content/") @@ -182,7 +183,7 @@ (timbre/info (format "Showing version '%s' of page '%s'" version page)) (layout/render "wiki.html" (merge (util/standard-params request) - {:title (str (:vers-col-hdr layout/config) " " version " of " page) + {:title (str (:vers-col-hdr util/config) " " version " of " page) :page page :content (md->html content)})))) @@ -191,14 +192,14 @@ "Render a diff between two versions of a page" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title layout/config))) + page (url-decode (or (:page params) (:default-page-title util/config))) version (:version params) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] (timbre/info (format "Showing diff between version '%s' of page '%s' and current" version page)) (layout/render "wiki.html" (merge (util/standard-params request) - {:title (str (:diff-title-prefix layout/config)" " version " of " page) + {:title (str (:diff-title-prefix util/config)" " version " of " page) :page page :content (d2h/diff2html (hist/diff repo-path file-name version))})))) @@ -213,7 +214,7 @@ user (session/get :user) redirect-to (or (:redirect-to params) "/wiki")] (cond - (= action (:logout-label layout/config)) + (= action (:logout-label util/config)) (do (timbre/info (str "User " user " logging out")) (session/remove! :user) @@ -225,7 +226,7 @@ true (layout/render "auth.html" (merge (util/standard-params request) - {:title (if user (str (:logout-link layout/config) " " user) (:login-link layout/config)) + {:title (if user (str (:logout-link util/config) " " user) (:login-link util/config)) :redirect-to ((:headers request) "referer")}))))) @@ -242,14 +243,14 @@ (auth/change-pass user oldpass pass2))] (layout/render "passwd.html" (merge (util/standard-params request) - {:title (str (:chpass-title-prefix layout/config) " " user) - :message (if changed? (:chpass-success layout/config)) + {:title (str (:chpass-title-prefix util/config) " " user) + :message (if changed? (:chpass-success util/config)) :error (cond (nil? oldpass) nil changed? nil - (< (count pass1) 8) (:chpass-too-short layout/config) - (not (= pass1 pass2)) (:chpass-bad-match layout/config) - true (:chpass-fail layout/config))})))) + (< (count pass1) 8) (:chpass-too-short util/config) + (not (= pass1 pass2)) (:chpass-bad-match util/config) + true (:chpass-fail util/config))})))) (defroutes wiki-routes diff --git a/src/smeagol/util.clj b/src/smeagol/util.clj index 6570b37..fd8f846 100644 --- a/src/smeagol/util.clj +++ b/src/smeagol/util.clj @@ -29,6 +29,14 @@ ;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; the relative path to the config file. +(def config-file-path (str (io/resource-path) "../config.edn")) + + +(def config (read-string (slurp config-file-path))) + + (defn standard-params "Return a map of standard parameters to pass to the template renderer." [request] From 8699b7cff40877276f412b71ca7fd041cbd35336 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 8 Aug 2017 15:50:33 +0100 Subject: [PATCH 11/14] Internationalisation support ... but no actual translation files yet. --- project.clj | 5 +- resources/config.edn | 88 +++--------------------------------- resources/i18n/en-GB.edn | 1 - src/smeagol/formatting.clj | 7 +-- src/smeagol/layout.clj | 2 +- src/smeagol/routes/admin.clj | 14 +++--- src/smeagol/routes/wiki.clj | 38 ++++++++-------- src/smeagol/util.clj | 25 ++++++++++ 8 files changed, 67 insertions(+), 113 deletions(-) diff --git a/project.clj b/project.clj index 7ae8b81..e8ea4ed 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,7 @@ [org.clojure/core.memoize "0.5.9"] [org.clojure/data.json "0.2.6"] [org.clojure/tools.logging "0.4.0"] + [org.clojars.simon_brooke/internationalisation "1.0.0"] [clj-jgit "0.8.9"] [clj-yaml "0.4.0"] [com.cemerick/url "0.1.1"] @@ -15,6 +16,7 @@ [crypto-password "0.2.0"] [environ "1.1.0"] [im.chit/cronj "1.4.4"] + [instaparse "1.4.1"] [lib-noir "0.9.9" :exclusions [org.clojure/tools.reader]] [markdown-clj "0.9.99" :exclusions [com.keminglabs/cljx]] [noir-exception "0.2.5"] @@ -32,7 +34,8 @@ [lein-environ "1.0.0"] [lein-bower "0.5.1"] [lein-ancient "0.5.5" :exclusions [org.clojure/clojure org.clojure/data.xml]] - [lein-marginalia "0.7.1" :exclusions [org.clojure/clojure]]] + [lein-marginalia "0.7.1" :exclusions [org.clojure/clojure]] + [lein-codox "0.10.3"]] :bower-dependencies [[simplemde "1.11.2"] ;; [vega-embed "3.0.0-beta.19"] vega-embed currently not loaded from Bower because of ;; dependency conflict which will hopefully be resolved soon. diff --git a/resources/config.edn b/resources/config.edn index 3930d1c..698a159 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -22,87 +22,13 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; config.edn: a simple configuration map for Smeagol; inspired by Cryogen. -;;; This is essentially all the text in the chrome - that which isn't editable -;;; through the wiki itself +;;; This is top-level configuration. ;; ; ; ; ; ; ; ; ; ; -{:add-user-label "Add new user" ;; label for the add user link on edit users page - :change-pass-label "Change password!" - ;; text of the change password widget itself on the - ;; change password page - :change-pass-link "Change password" - ;; text of the change password link on the menu - :change-pass-prompt "To change your password" - ;; text of the change password widget prompt on the - ;; change password page - :change-col-hdr "Changes" ;; header for the changes column in history - :chpass-bad-match "Your proposed passwords don't match" - ;; error text if proposed passwords don't match - :chpass-fail "Your password was not changed" - ;; error text on fail other htan too short or bad match - :chpass-success "Your password was changed" - ;; confirmation text on password change - :chpass-too-short "You proposed password wasn't long enough: eight characters required" - ;; error text if proposed password is too short - :chpass-title-prefix "Change password for" - ;; prefix for title of change password page - :cookies-about "About cookies" ;; about cookies text - :cookies-more "This website stores session information as a 'cookie' on your browser. This helps us show you the content you want to see. This cookie does not identify you, and cannot be read by other websites. It is deleted by your browser as soon as you leave this site. This website does not use any third party cookies, so your visit here cannot be tracked by other websites." - ;; more about cookies text - :default-page-title "Introduction" ;; title of the default page in this wiki - :del-col-hdr "Delete" ;; header for delete column on edit users page - :del-user-fail "Could not delete user" - ;; error message on failure to delete user - :del-user-success "Successfully deleted user" - ;; confirmation message on deletion of user - :diff-title-prefix "Changes since version" - ;; prefix for the header of the changes page - :edit-col-hdr "Edit" ;; header for edit column on edit users page - :edit-page-link "Edit this page" - ;; text of the edit page link on the content frame - :edit-title-prefix "Edit" ;; prefix for title of edit content page - :edit-users-link "Edit users" ;; text of the edit users link on the menu - :edit-users-title "Select user to edit" - ;; title of edit users page - :email-prompt "Email address" ;; text of the email widget prompt on edit user page - :file-upload-link-text "You may link to this file using a link of the form" - ;; Text introducing the link to an uploaded file - :file-upload-prompt "File to upload" ;; prompt string for the file upload widget - :file-upload-title "Upload a file" ;; title for the file upload page - :is-admin-prompt "Is administrator?" - :home-link "Home" ;; text of the home link on the menu - :login-label "Log in!" ;; text of the login widget on the login page - :login-link "Log in" ;; text of the login link on the menu - :login-prompt "To edit this wiki" - ;; text of the action widget prompt on the login page - :logout-label "Log out!" ;; text of the logout widget on the logout page - :logout-link "Log out" ;; text of the logout link on the menu - :logged-in-as "You are logged in as" - ;; text of the 'logged in as' label on the menu - :history-link "History" ;; text of the history link on the content frame - :history-title-prefix "History of" ;; prefix of the title on the history page - :new-pass-prompt "New password" ;; text of the new password widget prompt on the change - ;; password and edit user pages - :old-pass-prompt "Your password" - ;; text of the old password widget prompt on the change - ;; password page, and password widget on login page - :rpt-pass-prompt "And again" ;; text of the new password widget prompt on the change - ;; password and edit user pages - :save-prompt "When you have finished editing" - ;; text of the save widget label on edit content - ;; and edit user page - :save-label "Save!" ;; text of the save widget itself - :save-user-fail "Failed to store user" - :save-user-success "Successfully stored user" +{ :site-title "Smeagol" ;; overall title of the site, used in page headings - :username-prompt "Username" ;; text of the username widget prompt on edit user page - ;; text of the is admin widget prompt on edit user page - :user-title-prefix "Edit user" ;; prefix for title of edit user page - :vers-col-hdr "Version" ;; header for the version column in history - :what-col-hdr "What" ;; header for the what column in history - :what-changed-prompt "What have you changed?" - ;; text of the summary widget prompt on edit - ;; content page - :when-col-hdr "When" ;; header for the when column in history - :your-uname-prompt "Your username" ;; text of the username widget prompt on the login page - } + :default-locale "en-GB" ;; default language used for messages + :formatters {"vega" smeagol.formatting/process-vega + "vis" smeagol.formatting/process-vega + "mermaid" smeagol.formatting/process-mermaid} +} diff --git a/resources/i18n/en-GB.edn b/resources/i18n/en-GB.edn index a6de517..775818d 100644 --- a/resources/i18n/en-GB.edn +++ b/resources/i18n/en-GB.edn @@ -94,7 +94,6 @@ :save-label "Save!" ;; text of the save widget itself :save-user-fail "Failed to store user" :save-user-success "Successfully stored user" - :site-title "Smeagol" ;; overall title of the site, used in page headings :username-prompt "Username" ;; text of the username widget prompt on edit user page ;; text of the is admin widget prompt on edit user page :user-title-prefix "Edit user" ;; prefix for title of edit user page diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index daa5f66..49ea7d7 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -1,15 +1,16 @@ (ns ^{:doc "Format Semagol's enhanced markdown format." :author "Simon Brooke"} smeagol.formatting - (:require [clojure.string :as cs] + (:require [clojure.data.json :as json] + [clojure.string :as cs] [cemerick.url :refer (url url-encode url-decode)] + [clj-yaml.core :as yaml] [noir.io :as io] [noir.session :as session] [markdown.core :as md] [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] - [clj-yaml.core :as yaml] - [clojure.data.json :as json])) + )) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; diff --git a/src/smeagol/layout.clj b/src/smeagol/layout.clj index 20cc737..9b84fb4 100644 --- a/src/smeagol/layout.clj +++ b/src/smeagol/layout.clj @@ -42,7 +42,7 @@ (content-type (->> (assoc params (keyword (s/replace template #".html" "-selected")) "active" - :config util/config + :config (util/get-messages request) :dev (env :dev) :servlet-context (if-let [context (:servlet-context request)] diff --git a/src/smeagol/routes/admin.clj b/src/smeagol/routes/admin.clj index 64dc239..8cd5461 100644 --- a/src/smeagol/routes/admin.clj +++ b/src/smeagol/routes/admin.clj @@ -39,7 +39,7 @@ user (session/get :user)] (layout/render "edit-users.html" (merge (util/standard-params request) - {:title (:edit-users-title util/config) + {:title (:edit-users-title (util/get-messages request)) :users (auth/list-users)})))) (defn delete-user @@ -48,11 +48,11 @@ (let [params (keywordize-keys (:params request)) target (:target params) deleted (auth/delete-user target) - message (if deleted (str (:del-user-success util/config) " " target ".")) - error (if (not deleted) (str (:del-user-fail util/config) " " target "."))] + message (if deleted (str (:del-user-success (util/get-messages request)) " " target ".")) + error (if (not deleted) (str (:del-user-fail (util/get-messages request)) " " target "."))] (layout/render "edit-users.html" (merge (util/standard-params request) - {:title (:edit-users-title util/config) + {:title (:edit-users-title (util/get-messages request)) :message message :error error :users (auth/list-users)})))) @@ -67,9 +67,9 @@ password (if (and pass1 (auth/evaluate-password pass1 (:pass2 params))) pass1) stored (if (:email params) (auth/add-user target password (:email params) (:admin params))) - message (if stored (str (:save-user-success util/config) " " target ".")) + message (if stored (str (:save-user-success (util/get-messages request)) " " target ".")) error (if (and (:email params) (not stored)) - (str (:save-user-fail util/config) " " target ".")) + (str (:save-user-fail (util/get-messages request)) " " target ".")) details (auth/fetch-user-details target)] (if message (timbre/info message)) @@ -77,7 +77,7 @@ (timbre/warn error)) (layout/render "edit-user.html" (merge (util/standard-params request) - {:title (str (:edit-title-prefix util/config) " " target) + {:title (str (:edit-title-prefix (util/get-messages request)) " " target) :message message :error error :target target diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index 2a0fd0b..d7c4b17 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -57,7 +57,7 @@ (defn process-source "Process `source-text` and save it to the specified `file-path`, committing it to Git and finally redirecting to wiki-page." - [params suffix] + [params suffix request] (let [source-text (:src params) page (:page params) file-name (str page suffix) @@ -77,14 +77,14 @@ (if (= suffix ".md") (url-encode page) - (:default-page-title util/config)))))) + (util/get-message :default-page-title request)))))) (defn edit-page "Render a page in a text-area for editing. This could have been done in the same function as wiki-page, and that would have been neat, but I couldn't see how to establish security if that were done." ([request] - (edit-page request (:default-page-title util/config) ".md" "edit.html" "/content/_edit-side-bar.md")) + (edit-page request (util/get-message :default-page-title request) ".md" "edit.html" "/content/_edit-side-bar.md")) ([request default suffix template side-bar] (let [params (keywordize-keys (:params request)) src-text (:src params) @@ -95,11 +95,11 @@ (if (not exists?) (timbre/info (format "File '%s' not found; creating a new file" file-path)) (timbre/info (format "Opening '%s' for editing" file-path))) - (cond src-text (process-source params suffix) + (cond src-text (process-source params suffix request) true (layout/render template (merge (util/standard-params request) - {:title (str (:edit-title-prefix util/config) " " page) + {:title (str (util/get-message :edit-title-prefix request) " " page) :page page :side-bar (md->html (io/slurp-resource side-bar)) :content (if exists? (io/slurp-resource (str "/content/" page suffix)) "") @@ -117,7 +117,7 @@ [request] (timbre/debug (str "Request map: " request)) (let [params (keywordize-keys (:params request)) - page (or (:page params) (:default-page-title util/config)) + page (or (:page params) (util/get-message :default-page-title request)) file-name (str "/content/" page ".md") file-path (str (io/resource-path) file-name) exists? (.exists (clojure.java.io/as-file file-path))] @@ -138,7 +138,7 @@ if any. If none, error?" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title util/config))) + page (url-decode (or (:page params) (util/get-message :default-page-title request))) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] (timbre/info (format "Showing history of page '%s'" page)) @@ -157,7 +157,7 @@ uploaded (if upload (ul/store-upload params))] (layout/render "upload.html" (merge (util/standard-params request) - {:title (:file-upload-title util/config) + {:title (util/get-message :file-upload-title request) :uploaded uploaded :is-image (and uploaded @@ -175,7 +175,7 @@ "Render a specific historical version of a page" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title util/config))) + page (url-decode (or (:page params) (util/get-message :default-page-title request))) version (:version params) file-name (str page ".md") repo-path (str (io/resource-path) "/content/") @@ -183,7 +183,7 @@ (timbre/info (format "Showing version '%s' of page '%s'" version page)) (layout/render "wiki.html" (merge (util/standard-params request) - {:title (str (:vers-col-hdr util/config) " " version " of " page) + {:title (str (util/get-message :vers-col-hdr request) " " version " of " page) :page page :content (md->html content)})))) @@ -192,14 +192,14 @@ "Render a diff between two versions of a page" [request] (let [params (keywordize-keys (:params request)) - page (url-decode (or (:page params) (:default-page-title util/config))) + page (url-decode (or (:page params) (util/get-message :default-page-title request))) version (:version params) file-name (str page ".md") repo-path (str (io/resource-path) "/content/")] (timbre/info (format "Showing diff between version '%s' of page '%s' and current" version page)) (layout/render "wiki.html" (merge (util/standard-params request) - {:title (str (:diff-title-prefix util/config)" " version " of " page) + {:title (str (util/get-message :diff-title-prefix request)" " version " of " page) :page page :content (d2h/diff2html (hist/diff repo-path file-name version))})))) @@ -214,7 +214,7 @@ user (session/get :user) redirect-to (or (:redirect-to params) "/wiki")] (cond - (= action (:logout-label util/config)) + (= action (util/get-message :logout-label request)) (do (timbre/info (str "User " user " logging out")) (session/remove! :user) @@ -226,7 +226,7 @@ true (layout/render "auth.html" (merge (util/standard-params request) - {:title (if user (str (:logout-link util/config) " " user) (:login-link util/config)) + {:title (if user (str (util/get-message :logout-link request) " " user) (util/get-message :login-link request)) :redirect-to ((:headers request) "referer")}))))) @@ -243,14 +243,14 @@ (auth/change-pass user oldpass pass2))] (layout/render "passwd.html" (merge (util/standard-params request) - {:title (str (:chpass-title-prefix util/config) " " user) - :message (if changed? (:chpass-success util/config)) + {:title (str (util/get-message :chpass-title-prefix request) " " user) + :message (if changed? (util/get-message :chpass-success request)) :error (cond (nil? oldpass) nil changed? nil - (< (count pass1) 8) (:chpass-too-short util/config) - (not (= pass1 pass2)) (:chpass-bad-match util/config) - true (:chpass-fail util/config))})))) + (< (count pass1) 8) (util/get-message :chpass-too-short request) + (not (= pass1 pass2)) (util/get-message :chpass-bad-match request) + true (util/get-message :chpass-fail request))})))) (defroutes wiki-routes diff --git a/src/smeagol/util.clj b/src/smeagol/util.clj index fd8f846..d396902 100644 --- a/src/smeagol/util.clj +++ b/src/smeagol/util.clj @@ -3,6 +3,8 @@ smeagol.util (:require [noir.session :as session] [noir.io :as io] + [scot.weft.i18n.core :as i18n] + [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] [smeagol.formatting :refer [md->html]])) @@ -47,3 +49,26 @@ :header (md->html (io/slurp-resource "/content/_header.md")) :version (System/getProperty "smeagol.version")})) + +(defn raw-get-messages + "Return the most acceptable messages collection we have given the + `Accept-Language` header in this `request`." + [request] + (merge + (i18n/get-messages + ((:headers request) "accept-language") + (str (io/resource-path) "../i18n") + "en-GB") + config)) + + +(def get-messages (memoize raw-get-messages)) + +(defn get-message + [message-key request] + (let [messages (get-messages request)] + (if + (map? messages) + (or (messages message-key) message-key) + message-key))) + From 08c560fe2348b965982c4434db84b65460008684 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 8 Aug 2017 15:56:39 +0100 Subject: [PATCH 12/14] Internationalisation has now been moved into its own library. --- src/smeagol/i18n.clj | 53 -------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 src/smeagol/i18n.clj diff --git a/src/smeagol/i18n.clj b/src/smeagol/i18n.clj deleted file mode 100644 index 2e259a2..0000000 --- a/src/smeagol/i18n.clj +++ /dev/null @@ -1,53 +0,0 @@ -(ns ^{:doc "Internationalisation." - :author "Simon Brooke"} - smeagol.i18n - (:require [noir.session :as session] - [noir.io :as io] - [smeagol.authenticate :as auth] - [smeagol.formatting :refer [md->html]])) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;; -;;;; Smeagol: a very simple Wiki engine. -;;;; -;;;; This program is free software; you can redistribute it and/or -;;;; modify it under the terms of the GNU General Public License -;;;; as published by the Free Software Foundation; either version 2 -;;;; of the License, or (at your option) any later version. -;;;; -;;;; This program is distributed in the hope that it will be useful, -;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of -;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;;; GNU General Public License for more details. -;;;; -;;;; You should have received a copy of the GNU General Public License -;;;; along with this program; if not, write to the Free Software -;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -;;;; USA. -;;;; -;;;; Copyright (C) 2014 Simon Brooke -;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - - -(def i18n-cache {}) - - -(defn parse-accept-language - "The `Accept-Language` header is defined in section 14.4 of RFC 2616. It is composed - of a sequence of 'language tags' defined in section 3.10 of the same document. - Generally a language tag is a short alpha string (the 'primary language tag'), - optionally followed by a hyphen and another short alpha string (a 'sub tag'). - A sequence of more than one sub-tag is allowed. - - Generally a two-character primary tag will be an ISO-639 language code and any - initial two-character sub tag will be an ISO-3166 country code. - - Each language tag may optionally followed by a semi-colon followed by a 'q' value, - specified `q=0.8` where the numeric value is a real number in the range 0...1 - and represents the user's preference for this language (higher is better). If - no q value is supplied 1 is assumed. - - Language specifiers are separated by a comma followed by a space." - [accept-language-header] - ) From bf94fa8d7eddea2e9cc688fab7c4775f7ce435c6 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 8 Aug 2017 23:05:40 +0100 Subject: [PATCH 13/14] Genuinely pluggable/configurable formatter extensions! --- resources/public/content/Introduction.md | 50 +++++++++++++++--------- resources/public/content/stylesheet.css | 5 +++ resources/templates/wiki.html | 13 +----- src/smeagol/configuration.clj | 46 ++++++++++++++++++++++ src/smeagol/formatting.clj | 12 +----- src/smeagol/routes/wiki.clj | 1 - src/smeagol/util.clj | 9 +---- 7 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 src/smeagol/configuration.clj diff --git a/resources/public/content/Introduction.md b/resources/public/content/Introduction.md index b72c49b..feeb116 100644 --- a/resources/public/content/Introduction.md +++ b/resources/public/content/Introduction.md @@ -11,27 +11,19 @@ Smeagol is now a fully working small Wiki engine, and meets my own immediate nee ## Markup syntax Smeagol uses the Markdown format as provided by [markdown-clj](https://github.com/yogthos/markdown-clj), with the addition that anything enclosed in double square brackets, \[\[like this\]\], will be treated as a link into the wiki itself. -## Security and authentication -Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: +### Pluggable extensible markup - {:admin {:password "admin" :email "admin@localhost" :admin true} - :adam {:password "secret" :email "adam@localhost"}} +A system of pluggable, extensible formatters is supported. In normal markdown, code blocks may be delimited by three backticks at start and end, and often the syntax of the code can be indicated by a token immediately following the opening three backticks. This has been extended to allow custom formatters to be provided for such code blocks. Two example formatters are provided: -that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. +#### The Vega formatter -## Images -You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: +Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), the Vega formatter allows you to embed vega data visualisations into Smeagol pages. The graph description should start with a line comprising three back-ticks and then the word '`vega`', and end with a line comprising just three backticks. -![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) +Here's an example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): -## Now with data visualisation +##### Flight punctuality at London airports -Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), you can now embed visualisations into Smeagol pages, like this: - -### Flight punctuality at London airports - -Example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): -```vis +```vega data: url: "data/london.csv" transform: @@ -57,14 +49,13 @@ Note that this visualisation will not be rendered in the GitHub wiki, as it does ![Example visualisation](https://github.com/simon-brooke/smeagol/blob/develop/resources/public/data/london.png?raw=true) -## Now with embedded graphs +#### The Mermaid formatter -Graphs can now be embedded in a page using the [Mermaid](http://knsv.github.io/mermaid/index.html) graph description language. The graph description should start with a line comprising three back-ticks and then the -word 'mermaid', and end with a line comprising just three backticks. +Graphs can now be embedded in a page using the [Mermaid](http://knsv.github.io/mermaid/index.html) graph description language. The graph description should start with a line comprising three back-ticks and then the word `mermaid`, and end with a line comprising just three backticks. Here's an example culled from the Mermaid documentation. -### GANTT Chart +##### GANTT Chart ```mermaid gantt @@ -84,6 +75,27 @@ gantt Add to mermaid :1d ``` +To add your own formatter, compile it into a jar file which is on the classpath - it does *not* have to be part of the Smeagol project directly, and then edit the value of the key `:formatters` in the file `config.edn`; whose standard definition is: + + :formatters {"vega" smeagol.formatting/process-vega + "vis" smeagol.formatting/process-vega + "mermaid" smeagol.formatting/process-mermaid} + +The added key should be the word which will follow the opening three backticks of your code block, and the value of that key should be a symbol which evaluates to a function which can format the code block as required. + +## Security and authentication +Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: + + {:admin {:password "admin" :email "admin@localhost" :admin true} + :adam {:password "secret" :email "adam@localhost"}} + +that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. + +## Images +You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: + +![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) + ## Advertisement If you like what you see here, I am available for work on open source Clojure projects. diff --git a/resources/public/content/stylesheet.css b/resources/public/content/stylesheet.css index e8c2a16..37756d7 100644 --- a/resources/public/content/stylesheet.css +++ b/resources/public/content/stylesheet.css @@ -240,6 +240,11 @@ th { font-size: 66%; } +/* Add space between Vega-Embed links */ +.vega-actions a { + margin-right: 5px; +} + .warn { color: maroon; } diff --git a/resources/templates/wiki.html b/resources/templates/wiki.html index e87096c..afc2134 100644 --- a/resources/templates/wiki.html +++ b/resources/templates/wiki.html @@ -1,23 +1,14 @@ {% extends "templates/base.html" %} {% block extra-headers %} + {% style "vendor/mermaid/dist/mermaid.css" %} - {% script "/vendor/vega-lite/build/vega-lite.js" %} + {% script "vendor/vega-lite/build/vega-lite.js" %} - - - {% style "vendor/mermaid/dist/mermaid.css" %} {% script "vendor/mermaid/dist/mermaid.js" %} {% endblock %} diff --git a/src/smeagol/configuration.clj b/src/smeagol/configuration.clj new file mode 100644 index 0000000..1a17edc --- /dev/null +++ b/src/smeagol/configuration.clj @@ -0,0 +1,46 @@ +(ns ^{:doc "Read and make available configuration." + :author "Simon Brooke"} + smeagol.configuration + (:require [noir.io :as io])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: a very simple Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2017 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Right, doing the data visualisation thing is tricky. Doing it in the +;;;; pipeline doesn't work, because the md-to-html-string filter messes up +;;;; both YAML and JSON notation. So we need to extract the visualisation +;;;; fragments from the Markdown text and replace them with tokens we will +;;;; recognise afterwards, perform md-to-html-string, and then replace our +;;;; tokens with the transformed visualisation specification. +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(def config-file-path + "The relative path to the config file." + (str (io/resource-path) "../config.edn")) + + +(def config + "The actual configuration, as a map." + (read-string (slurp config-file-path))) diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index 49ea7d7..9fefcaf 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -10,7 +10,7 @@ [markdown.core :as md] [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] - )) + [smeagol.configuration :refer [config]])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; @@ -93,14 +93,6 @@ "\n")) -;; TODO: This isn't (yet) exactly what I want. The formatters ought to be configurable -;; without editing the Smeagol code directly. But it's a long way in the right direction. -(def ^:dynamic *formatters* - {"vega" process-vega - "vis" process-vega - "mermaid" process-mermaid}) - - (defn get-first-token "Return the first space-separated token of this `string`." [^String string] @@ -163,7 +155,7 @@ ([index result fragments processed] (let [fragment (first fragments) first-token (get-first-token fragment) - formatter (*formatters* first-token)] + formatter (eval ((:formatters config) first-token))] (cond (empty? fragments) (assoc result :text diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index d7c4b17..d0035d0 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -115,7 +115,6 @@ (defn wiki-page "Render the markdown page specified in this `request`, if any. If none found, redirect to edit-page" [request] - (timbre/debug (str "Request map: " request)) (let [params (keywordize-keys (:params request)) page (or (:page params) (util/get-message :default-page-title request)) file-name (str "/content/" page ".md") diff --git a/src/smeagol/util.clj b/src/smeagol/util.clj index d396902..94427b3 100644 --- a/src/smeagol/util.clj +++ b/src/smeagol/util.clj @@ -6,6 +6,7 @@ [scot.weft.i18n.core :as i18n] [taoensso.timbre :as timbre] [smeagol.authenticate :as auth] + [smeagol.configuration :refer [config]] [smeagol.formatting :refer [md->html]])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -32,13 +33,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; the relative path to the config file. -(def config-file-path (str (io/resource-path) "../config.edn")) - - -(def config (read-string (slurp config-file-path))) - - (defn standard-params "Return a map of standard parameters to pass to the template renderer." [request] @@ -64,6 +58,7 @@ (def get-messages (memoize raw-get-messages)) + (defn get-message [message-key request] (let [messages (get-messages request)] From 5634ec3e0fcfabac05d832410b61146098836461 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 8 Aug 2017 23:52:08 +0100 Subject: [PATCH 14/14] Upversioned; Embarrassing last minute regression --- README.md | 69 +++++++++++++++++++++++++++++-------- project.clj | 5 ++- src/smeagol/routes/wiki.clj | 2 +- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 068a5f3..feeb116 100644 --- a/README.md +++ b/README.md @@ -11,27 +11,19 @@ Smeagol is now a fully working small Wiki engine, and meets my own immediate nee ## Markup syntax Smeagol uses the Markdown format as provided by [markdown-clj](https://github.com/yogthos/markdown-clj), with the addition that anything enclosed in double square brackets, \[\[like this\]\], will be treated as a link into the wiki itself. -## Security and authentication -Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: +### Pluggable extensible markup - {:admin {:password "admin" :email "admin@localhost" :admin true} - :adam {:password "secret" :email "adam@localhost"}} +A system of pluggable, extensible formatters is supported. In normal markdown, code blocks may be delimited by three backticks at start and end, and often the syntax of the code can be indicated by a token immediately following the opening three backticks. This has been extended to allow custom formatters to be provided for such code blocks. Two example formatters are provided: -that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. +#### The Vega formatter -## Images -You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: +Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), the Vega formatter allows you to embed vega data visualisations into Smeagol pages. The graph description should start with a line comprising three back-ticks and then the word '`vega`', and end with a line comprising just three backticks. -![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) +Here's an example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): -## Now with data visualisation +##### Flight punctuality at London airports -Inspired by [visdown](http://visdown.amitkaps.com/) and [vega-lite](https://vega.github.io/vega-lite/docs/), you can now embed visualisations into Smeagol pages, like this: - -### Flight punctuality at London airports - -Example cribbed in its entirety from [here](http://visdown.amitkaps.com/london): -```vis +```vega data: url: "data/london.csv" transform: @@ -57,6 +49,53 @@ Note that this visualisation will not be rendered in the GitHub wiki, as it does ![Example visualisation](https://github.com/simon-brooke/smeagol/blob/develop/resources/public/data/london.png?raw=true) +#### The Mermaid formatter + +Graphs can now be embedded in a page using the [Mermaid](http://knsv.github.io/mermaid/index.html) graph description language. The graph description should start with a line comprising three back-ticks and then the word `mermaid`, and end with a line comprising just three backticks. + +Here's an example culled from the Mermaid documentation. + +##### GANTT Chart + +```mermaid +gantt + dateFormat YYYY-MM-DD + title Adding GANTT diagram functionality to mermaid + section A section + Completed task :done, des1, 2014-01-06,2014-01-08 + Active task :active, des2, 2014-01-09, 3d + Future task : des3, after des2, 5d + Future task2 : des4, after des3, 5d + section Critical tasks + Completed task in the critical line :crit, done, 2014-01-06,24h + Implement parser and jison :crit, done, after des1, 2d + Create tests for parser :crit, active, 3d + Future task in critical line :crit, 5d + Create tests for renderer :2d + Add to mermaid :1d +``` + +To add your own formatter, compile it into a jar file which is on the classpath - it does *not* have to be part of the Smeagol project directly, and then edit the value of the key `:formatters` in the file `config.edn`; whose standard definition is: + + :formatters {"vega" smeagol.formatting/process-vega + "vis" smeagol.formatting/process-vega + "mermaid" smeagol.formatting/process-mermaid} + +The added key should be the word which will follow the opening three backticks of your code block, and the value of that key should be a symbol which evaluates to a function which can format the code block as required. + +## Security and authentication +Security is now greatly improved. There is a file called *passwd* in the *resources* directory, which contains a clojure map which maps usernames to maps with plain-text passwords and emails thus: + + {:admin {:password "admin" :email "admin@localhost" :admin true} + :adam {:password "secret" :email "adam@localhost"}} + +that is to say, the username is a keyword and the corresponding password is a string. However, since version 0.5.0, users can now change their own passwords, and when the user changes their password their new password is encrypted using the [scrypt](http://www.tarsnap.com/scrypt.html) one-way encryption scheme. The password file is now no longer either in the *resources/public* directory so cannot be downloaded through the browser, nor in the git archive to which the Wiki content is stored, so that even if that git archive is remotely clonable an attacker cannot get the password file that way. + +## Images +You can (if you're logged in) upload files, including images, using the **Upload a file** link on the top menu bar. You can link to an uploaded image, or other images already available on the web, like this: + +![Smeagol](http://vignette3.wikia.nocookie.net/lotr/images/e/e1/Gollum_Render.png/revision/latest?cb=20141218075509) + ## Advertisement If you like what you see here, I am available for work on open source Clojure projects. diff --git a/project.clj b/project.clj index e8ea4ed..c526350 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject smeagol "0.5.1-SNAPSHOT" +(defproject smeagol "1.0.0-rc1" :description "A simple Git-backed Wiki inspired by Gollum" :url "https://github.com/simon-brooke/smeagol" :dependencies [[org.clojure/clojure "1.8.0"] @@ -16,7 +16,6 @@ [crypto-password "0.2.0"] [environ "1.1.0"] [im.chit/cronj "1.4.4"] - [instaparse "1.4.1"] [lib-noir "0.9.9" :exclusions [org.clojure/tools.reader]] [markdown-clj "0.9.99" :exclusions [com.keminglabs/cljx]] [noir-exception "0.2.5"] @@ -26,7 +25,7 @@ [org.slf4j/jcl-over-slf4j "1.7.25"] [prone "1.1.4"] [ring-server "0.4.0"] - [selmer "1.10.9"]] + [selmer "1.11.0"]] :repl-options {:init-ns smeagol.repl} :jvm-opts ["-server"] diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index d0035d0..e92313e 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -145,7 +145,7 @@ (merge (util/standard-params request) {:title (str "History of " page) :page page - :history (md->html (hist/find-history repo-path file-name))})))) + :history (hist/find-history repo-path file-name)})))) (defn upload-page "Render a form to allow the upload of a file."