diff --git a/.gitignore b/.gitignore index 95c58ff..ebbee66 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ smeagol.log* resources/public/content/uploads/ + +.eastwood diff --git a/project.clj b/project.clj index 09a9e65..c264426 100644 --- a/project.clj +++ b/project.clj @@ -46,6 +46,7 @@ [lein-codox "0.10.3"] [io.sarnowski/lein-docker "1.0.0"] [lein-environ "1.0.0"] + [lein-kibit "0.1.6"] [lein-marginalia "0.7.1" :exclusions [org.clojure/clojure]] [lein-npm "0.6.2"] [lein-ring "0.12.5" :exclusions [org.clojure/clojure]]] @@ -56,7 +57,8 @@ [vega-lite "4.1.1"] [mermaid "8.4.6"] [photoswipe "4.1.3"] - [tablesort "5.2.0"]] + [tablesort "5.2.0"] + [geocsv-js "simon-brooke/geocsv-js#80e5198"]] :root "resources/public/vendor"} :docker {:image-name "simonbrooke/smeagol" diff --git a/resources/config.edn b/resources/config.edn index 96ab5ca..c6a66ae 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -37,6 +37,15 @@ {:backticks {:formatter "smeagol.formatting/process-backticks" :scripts {} :styles {}} + :geocsv {:formatter "smeagol.extensions.geocsv/process-geocsv" + :scripts {:core {:local "vendor/node_modules/geocsv-js/js/geocsv.js"} + :leaflet {:local "vendor/node_modules/leaflet/dist/leaflet.js" + :remote "https://unpkg.com/leaflet@1.6.0/dist/leaflet.js"} + :papaparse {:local "vendor/node_modules/papaparse/papaparse.js" + :remote "https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.1.0/papaparse.min.js"}} + :styles {:leaflet {:local "vendor/node_modules/leaflet/dist/leaflet.css" + :remote "https://unpkg.com/leaflet@1.6.0/dist/leaflet.css"}} + :icon-url-base "uploads/map-pin/"} :mermaid {:formatter "smeagol.extensions.mermaid/process-mermaid" :scripts {:core {:local "vendor/node_modules/mermaid/dist/mermaid.min.js" :remote "https://cdnjs.cloudflare.com/ajax/libs/mermaid/8.4.6/mermaid.min.js"}}} @@ -68,6 +77,7 @@ ;; stored in the /small directory :med 400 ;; maximum dimension of thumbnails ;; stored in the /med directory + :map-pin 40 ;; stored in the /map-pin directory ;; you can add as many extra keys and values as ;; you like here for additional sizes of images. ;; Images will only be scaled if their maximum diff --git a/resources/public/content/Extensible Markup.md b/resources/public/content/Extensible Markup.md index 41c9873..5a91d7f 100644 --- a/resources/public/content/Extensible Markup.md +++ b/resources/public/content/Extensible Markup.md @@ -1,6 +1,77 @@ The basic format of Smeagol pages is [Markdown](https://daringfireball.net/projects/markdown/); documentation on how to format them is [here](https://daringfireball.net/projects/markdown/syntax). Note that there are a number of slightly different variants of Markdown; the version used by Smeagol does not currently allow tables. -A system of pluggable, extensible formatters is supported. In normal markdown, code blocks may be delimited by three backticks at start and end, and often the syntax of the code can be indicated by a token immediately following the opening three backticks. This has been extended to allow custom formatters to be provided for such code blocks. Two example formatters are provided: +A system of pluggable, extensible formatters is supported. In normal markdown, code blocks may be delimited by three backticks at start and end, and often the syntax of the code can be indicated by a token immediately following the opening three backticks. This has been extended to allow custom formatters to be provided for such code blocks. Example formatters are provided: + +## The GeoCSV formatter + +Allows you to embed maps with markers on them into Wiki pages. To embed a map, start with a line comprising three backticks and the word '`geocsv`', then as many lines as you like with comma-separated values (CSV) data to show on your map, followed by a line comprising just three backticks and nothing else. + +The CSV must have the column titles in the first line, and must have columns called `latitude` and `longitude`, which must both contain floating point numbers, not degrees, minutes and seconds. The CSV *may* also have a column called `name`, which will be used as the heading for the popup that shows if you click an image, and a column named category. If the category column is present, and contains a value in a given row, the marker image for the marker for that row will the uploaded file whose name is that value followed by `-pin.png`. + +Here's an example: the capital cities of Europe. + +```geocsv +Country,Name,Latitude,Longitude,CountryCode,Continent,Category +Aland Islands,Mariehamn,60.116667,19.9,AX,Europe, +Albania,Tirana,41.3166666666667,19.816667,AL,Europe,AL +Andorra,Andorra la Vella,42.5,1.516667,AD,Europe,AD +Armenia,Yerevan,40.1666666666667,44.5,AM,Europe,AM +Austria,Vienna,48.2,16.366667,AT,Europe,AT +Azerbaijan,Baku,40.3833333333333,49.866667,AZ,Europe,AZ +Belarus,Minsk,53.9,27.566667,BY,Europe,BY +Belgium,Brussels,50.8333333333333,4.333333,BE,Europe,BE +Bosnia and Herzegovina,Sarajevo,43.8666666666667,18.416667,BA,Europe,BA +Bulgaria,Sofia,42.6833333333333,23.316667,BG,Europe,BG +Croatia,Zagreb,45.8,16,HR,Europe,HR +Cyprus,Nicosia,35.1666666666667,33.366667,CY,Europe,CY +Czech Republic,Prague,50.0833333333333,14.466667,CZ,Europe,CZ +Denmark,Copenhagen,55.6666666666667,12.583333,DK,Europe,DK +Estonia,Tallinn,59.4333333333333,24.716667,EE,Europe,EE +Faroe Islands,Torshavn,62,-6.766667,FO,Europe, +Finland,Helsinki,60.1666666666667,24.933333,FI,Europe,FI +France,Paris,48.8666666666667,2.333333,FR,Europe,FR +Georgia,Tbilisi,41.6833333333333,44.833333,GE,Europe,GE +Germany,Berlin,52.5166666666667,13.4,DE,Europe,DE +Gibraltar,Gibraltar,36.1333333333333,-5.35,GI,Europe,GI +Greece,Athens,37.9833333333333,23.733333,GR,Europe,GR +Guernsey,Saint Peter Port,49.45,-2.533333,GG,Europe,GG +Vatican City,Vatican City,41.9,12.45,VA,Europe,VA +Hungary,Budapest,47.5,19.083333,HU,Europe,HU +[[Iceland]],Reykjavik,64.15,-21.95,IS,Europe,IS +Ireland,Dublin,53.3166666666667,-6.233333,IE,Europe,IE +Isle of Man,Douglas,54.15,-4.483333,IM,Europe,IM +Italy,Rome,41.9,12.483333,IT,Europe,IT +Jersey,Saint Helier,49.1833333333333,-2.1,JE,Europe,JE +Kosovo,Pristina,42.6666666666667,21.166667,KO,Europe, +Latvia,Riga,56.95,24.1,LV,Europe,LV +Liechtenstein,Vaduz,47.1333333333333,9.516667,LI,Europe,LI +Lithuania,Vilnius,54.6833333333333,25.316667,LT,Europe,LT +Luxembourg,Luxembourg,49.6,6.116667,LU,Europe,LU +Macedonia,Skopje,42,21.433333,MK,Europe,MK +Malta,Valletta,35.8833333333333,14.5,MT,Europe,MT +Moldova,Chisinau,47,28.85,MD,Europe,MD +Monaco,Monaco,43.7333333333333,7.416667,MC,Europe,MC +Montenegro,Podgorica,42.4333333333333,19.266667,ME,Europe,ME +Netherlands,Amsterdam,52.35,4.916667,NL,Europe,NL +Norway,Oslo,59.9166666666667,10.75,NO,Europe,NO +Poland,Warsaw,52.25,21,PL,Europe,PL +Portugal,Lisbon,38.7166666666667,-9.133333,PT,Europe,PT +Romania,Bucharest,44.4333333333333,26.1,RO,Europe,RO +Russia,Moscow,55.75,37.6,RU,Europe,RU +San Marino,San Marino,43.9333333333333,12.416667,SM,Europe,SM +Serbia,Belgrade,44.8333333333333,20.5,RS,Europe,RS +Slovakia,Bratislava,48.15,17.116667,SK,Europe,SK +Slovenia,Ljubljana,46.05,14.516667,SI,Europe,SI +Spain,Madrid,40.4,-3.683333,ES,Europe,ES +Svalbard,Longyearbyen,78.2166666666667,15.633333,SJ,Europe, +Sweden,Stockholm,59.3333333333333,18.05,SE,Europe,SE +Switzerland,Bern,46.9166666666667,7.466667,CH,Europe,CH +Turkey,Ankara,39.9333333333333,32.866667,TR,Europe,TR +Ukraine,Kyiv,50.4333333333333,30.516667,UA,Europe,UA +United Kingdom,London,51.5,-0.083333,GB,Europe,GB +Northern Cyprus,North Nicosia,35.183333,33.366667,NULL,Europe, + +``` ## The Vega formatter diff --git a/resources/templates/base.html b/resources/templates/base.html index 48e5ae4..d36cb27 100644 --- a/resources/templates/base.html +++ b/resources/templates/base.html @@ -5,6 +5,7 @@ + {% style "/content/stylesheet.css" %} {% block extra-headers %} {% endblock %} diff --git a/src/smeagol/extensions/geocsv.clj b/src/smeagol/extensions/geocsv.clj new file mode 100644 index 0000000..925fd50 --- /dev/null +++ b/src/smeagol/extensions/geocsv.clj @@ -0,0 +1,55 @@ +(ns ^{:doc "GeoCSV extension for Semagol's extendsible markdown format." + :author "Simon Brooke"} + smeagol.extensions.geocsv + (:require [smeagol.configuration :refer [config]] + [smeagol.extensions.utils :refer :all] + [taoensso.timbre :as log])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;; +;;;; Smeagol: an extensible Wiki engine. +;;;; +;;;; This program is free software; you can redistribute it and/or +;;;; modify it under the terms of the GNU General Public License +;;;; as published by the Free Software Foundation; either version 2 +;;;; of the License, or (at your option) any later version. +;;;; +;;;; This program is distributed in the hope that it will be useful, +;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;;; GNU General Public License for more details. +;;;; +;;;; You should have received a copy of the GNU General Public License +;;;; along with this program; if not, write to the Free Software +;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +;;;; USA. +;;;; +;;;; Copyright (C) 2017 Simon Brooke +;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn process-geocsv + "If this `url-or-geo-csv` is a valid URL, it is assumed to point to a CSV file + containing geographical point data; otherwise, it is expected to be CSV formatted + text with at least `latitude` and `longitude` columns." + [^String url-or-geo-csv ^Integer index] + (let [data (resource-url-or-data->data url-or-geo-csv) + geo-csv (:data data)] + (log/info "Retrieved geo-csv from " (:from data) " `" ((:from data) data) "`") + (str "\n
\n
\n"
+         geo-csv
+         "\n
+
+ + "))) diff --git a/src/smeagol/formatting.clj b/src/smeagol/formatting.clj index eeb0753..f7de5e7 100644 --- a/src/smeagol/formatting.clj +++ b/src/smeagol/formatting.clj @@ -7,6 +7,7 @@ [clj-yaml.core :as yaml] [markdown.core :as md] [smeagol.configuration :refer [config]] + [smeagol.extensions.geocsv :refer [process-geocsv]] [smeagol.extensions.mermaid :refer [process-mermaid]] [smeagol.extensions.photoswipe :refer [process-photoswipe]] [smeagol.extensions.vega :refer [process-vega]] @@ -93,7 +94,7 @@ (merge-with deep-merge v1 v2) v2))] (if (some identity vs) - (reduce #(rec-merge %1 %2) v vs) + (reduce rec-merge v vs) (last vs)))) diff --git a/src/smeagol/middleware.clj b/src/smeagol/middleware.clj index 4ca288d..7d8c4bf 100644 --- a/src/smeagol/middleware.clj +++ b/src/smeagol/middleware.clj @@ -53,8 +53,8 @@ #(wrap-resource % "public") #(wrap-file % util/content-dir {:index-files? false :prefer-handler? true}) - #(wrap-content-type %) - #(wrap-not-modified %)]) + wrap-content-type + wrap-not-modified]) (defn load-middleware [] diff --git a/src/smeagol/routes/wiki.clj b/src/smeagol/routes/wiki.clj index a691b2f..6695c36 100644 --- a/src/smeagol/routes/wiki.clj +++ b/src/smeagol/routes/wiki.clj @@ -325,9 +325,9 @@ (if-not (empty? uploaded) (do - (map - #(git/git-add git-repo (str :resource %)) - (remove nil? uploaded)) + (doall (map + #(git/git-add git-repo (str :resource %)) + (remove nil? uploaded))) (git/git-commit git-repo summary {:name user :email (auth/get-email user)}))) (layout/render "upload.html" (merge (util/standard-params request) diff --git a/src/smeagol/uploads.clj b/src/smeagol/uploads.clj index a6912e0..d064acb 100644 --- a/src/smeagol/uploads.clj +++ b/src/smeagol/uploads.clj @@ -4,6 +4,7 @@ (:require [clojure.string :as cs] [clojure.java.io :as io] [image-resizer.core :refer [resize]] + [image-resizer.scale-methods :as sm] [image-resizer.util :refer :all] [me.raynes.fs :as fs] [noir.io :as nio] @@ -58,18 +59,23 @@ :format - :gif, :jpg, :png or anything supported by ImageIO :quality - for JPEG images, a number between 0 and 100" [^RenderedImage img dest & {:keys [format quality] :or {format :jpg}}] - (log/info "Writing to " dest) + (log/info "Writing as" format "to" dest) (let [fmt (subs (fs/extension (cs/lower-case dest)) 1) iw (doto ^ImageWriter (first (iterator-seq (ImageIO/getImageWritersByFormatName fmt))) (.setOutput (FileImageOutputStream. (io/file dest)))) - iw-param (doto ^ImageWriteParam (.getDefaultWriteParam iw) - (.setCompressionMode ImageWriteParam/MODE_EXPLICIT) - (.setCompressionQuality (float (/ (or quality 75) 100)))) + iw-param (case format + :jpg (doto ^ImageWriteParam (.getDefaultWriteParam iw) + (.setCompressionMode ImageWriteParam/MODE_EXPLICIT) + (.setCompressionQuality (float (/ (or quality 75) 100)))) + (:png :gif) nil) iio-img (IIOImage. img nil nil)] - (.write iw nil iio-img iw-param))) + (log/info "smeagol.uploads/write-image: fmt=" fmt "format=" format) + (if iw + (.write iw nil iio-img iw-param) + (log/error "smeagol.uploads/write-image: no suitable writer found")))) (def image? "True if the file at this `filename` appears as though it may be an image" @@ -90,15 +96,22 @@ (keys (config :thumbnails)))) (log/info filename " cannot be thumbnailed."))) ([^String path ^String filename size ^RenderedImage image] - (let [s (-> config :thumbnails size) - d (dimensions image) - p (io/file path (name size) filename)] - (if (and (integer? s) (some #(> % s) d)) - (do - (write-image (resize image s s) p) - (log/info "Created a " size " thumbnail of " filename) - {:size size :filename filename :location (str p) :is-image true}) - (log/info filename "is smaller than " s "x" s " and was not scaled to " size))))) + (when image + (try + (let [s (-> config :thumbnails size) + d (dimensions image) + p (io/file path (name size) filename)] + (if (and (integer? s) (some #(> % s) d)) + (do + (write-image + (resize image s s) + p + :format (keyword (subs (fs/extension filename) 1))) + (log/info "Created a " size " thumbnail of " filename) + {:size size :filename filename :location (str p) :is-image true}) + (log/info filename "is smaller than " s "x" s " and was not scaled to " size))) + (catch Exception any + (log/error "Failed to thumbnail image " filename "to size" size ":" any)))))) (defn store-upload "Store an upload both to the file system and to the database. diff --git a/test/smeagol/test/local_links.clj b/test/smeagol/test/local_links.clj index e02d727..31dc051 100644 --- a/test/smeagol/test/local_links.clj +++ b/test/smeagol/test/local_links.clj @@ -9,6 +9,9 @@ (is (= (local-links nil) no-text-error) "Should NOT fail with a no pointer exception!") (is (= (local-links "") "") "Empty string should pass through unchanged.") (is (= (local-links "[[froboz]]") "froboz") "Local link should be rewritten.") + (is (= (local-links "[[Iceland]],Reykjavik,64.15,-21.95,IS,Europe,IS") + "Iceland,Reykjavik,64.15,-21.95,IS,Europe,IS") " + There should be no new carriage return after a local link") (let [text (str "# This is a heading" "[This is a foreign link](http://to.somewhere)")] (is (= (local-links text) text) "Foreign links should be unchanged"))