From a5881c3b9789f2cefefc7c662754dbc7238f8f3e Mon Sep 17 00:00:00 2001 From: Simon Brooke Date: Tue, 28 Jan 2020 22:32:49 +0000 Subject: [PATCH] Now successfully remote-loading data from Google sheets --- README.md | 44 +++--- env/dev/resources/config.edn | 3 +- pom.xml.asc | 11 ++ resources/html/home.html | 45 +++--- resources/public/css/spinner.css | 141 +++++++++++++++++++ src/clj/geocsv/handler.clj | 57 +++++--- src/clj/geocsv/routes/home.clj | 28 ++-- src/clj/geocsv/routes/{json.clj => rest.clj} | 27 ++-- src/cljs/geocsv/events.cljs | 2 +- src/cljs/geocsv/views/map.cljs | 2 +- 10 files changed, 273 insertions(+), 87 deletions(-) create mode 100644 pom.xml.asc create mode 100644 resources/public/css/spinner.css rename src/clj/geocsv/routes/{json.clj => rest.clj} (79%) diff --git a/README.md b/README.md index af49f3d..2e12b74 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,30 @@ The CSV file must have Additionally, the value of the column `category`, if present, will be used to select map pins from the map pins folder, if a suitable pin is present. Thus is the value of `category` is `foo`, a map pin image with the name `Foo-pin.png` will be selected. +## Passing CSV files to the app + +### Loading them onto the server + +If you run the server running **geocsv**, the simplest way to add CSV files is simply to copy them into the directory `resourcs/data`. The default file is the one named `data.csv`, which is the one that will be served if nothing else is specified. Other files can be specifiec by appending `?file=filename` to the URL; so if the URL of your geocsv service is + + https://geocsv.example.com/ + +and the file you want to view is `myfile.csv`, then you would specify this as + + https://geocsv.example.com/?file=myfile.csv + +### Using a Google spreadsheet + +If you use [Google Sheets](https://www.google.co.uk/sheets/about/), then every sheet has a 'document id', a long string of characters which uniquely identifies that sheet. Suppose your Google spreadsheet has a document id of `abcdefghijklmnopqrstuvwxyz-12345`, then you could pull data from this spreadsheet by specifying: + + https://geocsv.example.com/?docid=abcdefghijklmnopqrstuvwxyz-12345 + +The spreadsheet **must** be publicly readable. + ## Not yet working GeoCSV is at an early stage of development, and some features are not yet working. -### Doesn't actually interpret CSV - -I haven't yet found an easy way to parse CSV into EDN client side, so I've written a [separate library](https://github.com/simon-brooke/csv2edn) to do it server side. However, that library is not yet integrated. Currently the client side actually interprets JSON. - ### Missing map pin images At the current stage of development, if no appropriate image exists in the `resources/public/img/map-pins` folder, that's your problem. **TODO:** I intend at some point to make missing pin images default to `unknown-pin.png`, which does exist. @@ -28,26 +44,6 @@ At the current stage of development, if no appropriate image exists in the `reso Currently the map is initially centred roughly on the centre of Scotland, and scaled arbitrarily. It should compute an appropriate centre and scale from the data provided, but currently doesn't. -### There's no way of linking your own data feed - -Currently, the data is taken from the file `resources/public/data/data.json`. What I intend is that you should have a form which allows you to either - -1. enter [the `DOCID` of your own (publicly readable) Google Sheets spreadsheet](https://stackoverflow.com/questions/33713084/download-link-for-google-spreadsheets-csv-export-with-multiple-sheets); -2. enter the URL of a CSV file publicly available on the web; -3. upload a CSV file to the server. - -### There's no way of shareing the map of your own data with other people - -Currently, the data that is shared is just the data that's present when the app is compiled. Ideally, there should be a way of generating a URL, which might take the form: - - https://server.name/geocsv/docid/564747867 - -To show data from the first sheet of the Google Sheets spreadsheet whose `DOCID` is 564747867; or - - https://server.name/geocsv?uri=https://address.of.another.server/path/to/csv-file.csv - -to show the content of a publicly available CSV file. - ## Prerequisites You will need [Leiningen][1] 2.0 or above installed. diff --git a/env/dev/resources/config.edn b/env/dev/resources/config.edn index 0967ef4..8df4a7c 100644 --- a/env/dev/resources/config.edn +++ b/env/dev/resources/config.edn @@ -1 +1,2 @@ -{} +{:prod false + :port 3000} diff --git a/pom.xml.asc b/pom.xml.asc new file mode 100644 index 0000000..1b6b04c --- /dev/null +++ b/pom.xml.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP SIGNATURE----- + +iQEzBAABCAAdFiEEgk5pWlj1vvTbfWCZp6TxjR1N+YcFAl4wte0ACgkQp6TxjR1N ++YdU8wgA3OSP8XYP/mI5EAmuR11ryXwTspVgbwBxIK82m+xDwTLQE9Za0I4FdS+g +RGEzo+S5LiYG1dD5fXkmtNr4WcnUJ7v5TFJZWE9eBAX3De0uVsksfF5u7F7I5oGy +mciv/C8hxyM6AQE9OviWE5UtDbJT1NxaLmLZwiZrnG2uq5JDR7h+7N4exWM99g7u +1FpHs/ed3ui+IIgsJdBnHsUBh8eOogvVmPpPePRIE8lQxoGzX/BTkZT3ed9xmdIF +ruyXwgVc9tMzy2wYu05kbOCcYqGeJK94UtPUhOmPbs3Z6nGRaVH5zQb3kiRo4P+S +KlAZkC9+sLY1mACD4JVjV1EWMAYOHg== +=GbQE +-----END PGP SIGNATURE----- diff --git a/resources/html/home.html b/resources/html/home.html index 770c5cc..9cb562e 100644 --- a/resources/html/home.html +++ b/resources/html/home.html @@ -1,31 +1,32 @@ - - - Welcome to geocsv + + + Welcome to geocsv -
-
-
-
-

Welcome to geocsv

-

If you're seeing this message, that means you haven't yet compiled your ClojureScript!

-

Please run lein figwheel to start the ClojureScript compiler and reload the page.

-

For better ClojureScript development experience in Chrome follow these steps:

-
    -
  • Open DevTools -
  • Go to Settings ("three dots" icon in the upper right corner of DevTools > Menu > Settings F1 > General > Console) -
  • Check-in "Enable custom formatters" -
  • Close DevTools -
  • Open DevTools -
-

See ClojureScript documentation for further details.

-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+

+ geocsv is loading. + You must enable JavaScript to use geocsv. +

{% block foot %} @@ -49,7 +50,7 @@ {% style "/css/geocsv.css" %} diff --git a/resources/public/css/spinner.css b/resources/public/css/spinner.css new file mode 100644 index 0000000..979f4ff --- /dev/null +++ b/resources/public/css/spinner.css @@ -0,0 +1,141 @@ +/* + * Cribbed from http://tobiasahlin.com/spinkit/ + * (source here https://github.com/tobiasahlin/SpinKit) + * Thanks Tobias! + */ + +.sk-fading-circle { + margin: 100px auto; + width: 40px; + height: 40px; + position: relative; +} + +.sk-fading-circle .sk-circle { + width: 100%; + height: 100%; + position: absolute; + left: 0; + top: 0; +} + +.sk-fading-circle .sk-circle:before { + content: ''; + display: block; + margin: 0 auto; + width: 15%; + height: 15%; + background-color: #3298dc; + border-radius: 100%; + -webkit-animation: sk-circleFadeDelay 1.2s infinite ease-in-out both; + animation: sk-circleFadeDelay 1.2s infinite ease-in-out both; +} +.sk-fading-circle .sk-circle2 { + -webkit-transform: rotate(30deg); + -ms-transform: rotate(30deg); + transform: rotate(30deg); +} +.sk-fading-circle .sk-circle3 { + -webkit-transform: rotate(60deg); + -ms-transform: rotate(60deg); + transform: rotate(60deg); +} +.sk-fading-circle .sk-circle4 { + -webkit-transform: rotate(90deg); + -ms-transform: rotate(90deg); + transform: rotate(90deg); +} +.sk-fading-circle .sk-circle5 { + -webkit-transform: rotate(120deg); + -ms-transform: rotate(120deg); + transform: rotate(120deg); +} +.sk-fading-circle .sk-circle6 { + -webkit-transform: rotate(150deg); + -ms-transform: rotate(150deg); + transform: rotate(150deg); +} +.sk-fading-circle .sk-circle7 { + -webkit-transform: rotate(180deg); + -ms-transform: rotate(180deg); + transform: rotate(180deg); +} +.sk-fading-circle .sk-circle8 { + -webkit-transform: rotate(210deg); + -ms-transform: rotate(210deg); + transform: rotate(210deg); +} +.sk-fading-circle .sk-circle9 { + -webkit-transform: rotate(240deg); + -ms-transform: rotate(240deg); + transform: rotate(240deg); +} +.sk-fading-circle .sk-circle10 { + -webkit-transform: rotate(270deg); + -ms-transform: rotate(270deg); + transform: rotate(270deg); +} +.sk-fading-circle .sk-circle11 { + -webkit-transform: rotate(300deg); + -ms-transform: rotate(300deg); + transform: rotate(300deg); +} +.sk-fading-circle .sk-circle12 { + -webkit-transform: rotate(330deg); + -ms-transform: rotate(330deg); + transform: rotate(330deg); +} +.sk-fading-circle .sk-circle2:before { + -webkit-animation-delay: -1.1s; + animation-delay: -1.1s; +} +.sk-fading-circle .sk-circle3:before { + -webkit-animation-delay: -1s; + animation-delay: -1s; +} +.sk-fading-circle .sk-circle4:before { + -webkit-animation-delay: -0.9s; + animation-delay: -0.9s; +} +.sk-fading-circle .sk-circle5:before { + -webkit-animation-delay: -0.8s; + animation-delay: -0.8s; +} +.sk-fading-circle .sk-circle6:before { + -webkit-animation-delay: -0.7s; + animation-delay: -0.7s; +} +.sk-fading-circle .sk-circle7:before { + -webkit-animation-delay: -0.6s; + animation-delay: -0.6s; +} +.sk-fading-circle .sk-circle8:before { + -webkit-animation-delay: -0.5s; + animation-delay: -0.5s; +} +.sk-fading-circle .sk-circle9:before { + -webkit-animation-delay: -0.4s; + animation-delay: -0.4s; +} +.sk-fading-circle .sk-circle10:before { + -webkit-animation-delay: -0.3s; + animation-delay: -0.3s; +} +.sk-fading-circle .sk-circle11:before { + -webkit-animation-delay: -0.2s; + animation-delay: -0.2s; +} +.sk-fading-circle .sk-circle12:before { + -webkit-animation-delay: -0.1s; + animation-delay: -0.1s; +} + +@-webkit-keyframes sk-circleFadeDelay { + 0%, 39%, 100% { opacity: 0; } + 40% { opacity: 1; } +} + +@keyframes sk-circleFadeDelay { + 0%, 39%, 100% { opacity: 0; } + 40% { opacity: 1; } +} diff --git a/src/clj/geocsv/handler.clj b/src/clj/geocsv/handler.clj index 48c42d8..001fbf6 100644 --- a/src/clj/geocsv/handler.clj +++ b/src/clj/geocsv/handler.clj @@ -1,10 +1,11 @@ (ns geocsv.handler (:require [compojure.core :refer [routes wrap-routes]] + [compojure.route :as route] [geocsv.env :refer [defaults]] [geocsv.middleware :as middleware] [geocsv.layout :refer [error-page]] [geocsv.routes.home :refer [home-routes]] - [geocsv.routes.json :refer [json-routes]] + [geocsv.routes.rest :refer [rest-routes]] [reitit.ring :as ring] [ring.middleware.content-type :refer [wrap-content-type]] [ring.middleware.webjars :refer [wrap-webjars]] @@ -16,24 +17,42 @@ (mount/defstate app-routes :start - (ring/ring-handler - (ring/router - [(home-routes) - (-> #'json/json-routes - (wrap-routes middleware/wrap-csrf) - (wrap-routes middleware/wrap-formats))]) - (ring/routes - (ring/create-resource-handler - {:path "/"}) - (wrap-content-type - (wrap-webjars (constantly nil))) - (ring/create-default-handler - {:not-found - (constantly (error-page {:status 404, :title "404 - Page not found"})) - :method-not-allowed - (constantly (error-page {:status 405, :title "405 - Not allowed"})) - :not-acceptable - (constantly (error-page {:status 406, :title "406 - Not acceptable"}))})))) + ;; This is an older way of doing routing and Dmitri Sotnikov now does it + ;; another way which is almost certainly better but I can't make it work. + (routes + (-> #'home-routes + (wrap-routes middleware/wrap-csrf) + (wrap-routes middleware/wrap-formats)) + (-> #'rest-routes + (wrap-routes middleware/wrap-csrf) + (wrap-routes middleware/wrap-formats)) + (route/resources "/") + (route/not-found + (:body + (error-page {:status 404 + :title "Page not found" + :message "The page you requested has not yet been implemented"}))))) + + +;; (ring/ring-handler +;; (ring/router +;; [(home-routes) +;; ;; (-> rest-routes +;; ;; (wrap-routes middleware/wrap-csrf) +;; ;; (wrap-routes middleware/wrap-formats)) +;; ]) +;; (ring/routes +;; (ring/create-resource-handler +;; {:path "/"}) +;; (wrap-content-type +;; (wrap-webjars (constantly nil))) +;; (ring/create-default-handler +;; {:not-found +;; (constantly (error-page {:status 404, :title "404 - Page not found"})) +;; :method-not-allowed +;; (constantly (error-page {:status 405, :title "405 - Not allowed"})) +;; :not-acceptable +;; (constantly (error-page {:status 406, :title "406 - Not acceptable"}))})))) (defn app [] (middleware/wrap-base #'app-routes)) diff --git a/src/clj/geocsv/routes/home.clj b/src/clj/geocsv/routes/home.clj index fbfd090..005a18b 100644 --- a/src/clj/geocsv/routes/home.clj +++ b/src/clj/geocsv/routes/home.clj @@ -1,19 +1,29 @@ (ns geocsv.routes.home (:require [clojure.java.io :as io] + [compojure.core :refer [defroutes GET POST]] [geocsv.layout :as layout] [geocsv.middleware :as middleware] [ring.util.response] [ring.util.http-response :as response])) (defn home-page [request] - (layout/render request "home.html")) + "Serve the home page, in the process merging any parameters passed + in the request into the session." + (assoc + (layout/render request "home.html") + :session + (merge + (:session request) + (:params request)))) -(defn home-routes [] - ["" - {:middleware [middleware/wrap-csrf - middleware/wrap-formats]} - ["/" {:get home-page}] - ["/docs" {:get (fn [_] - (-> (response/ok (-> "docs/docs.md" io/resource slurp)) - (response/header "Content-Type" "text/plain; charset=utf-8")))}]]) +(defroutes home-routes + (GET "/" request (home-page request)) + (GET "/docs" _ (fn [_] + (-> + (response/ok + (-> + "docs/docs.md" + io/resource + slurp)) + (response/header "Content-Type" "text/plain; charset=utf-8"))))) diff --git a/src/clj/geocsv/routes/json.clj b/src/clj/geocsv/routes/rest.clj similarity index 79% rename from src/clj/geocsv/routes/json.clj rename to src/clj/geocsv/routes/rest.clj index b9e7d30..26d0446 100644 --- a/src/clj/geocsv/routes/json.clj +++ b/src/clj/geocsv/routes/rest.clj @@ -1,5 +1,5 @@ -(ns geocsv.routes.json - "JSON routes for geocsv." +(ns geocsv.routes.rest + "REST routes for geocsv." (:require [adl-support.core :as ac] [adl-support.rest-support :as ar] [clojure.core.memoize :as memo] @@ -44,8 +44,9 @@ (let [grammar-matcher (.getPathMatcher (java.nio.file.FileSystems/getDefault) "glob:*-pin.png")] - (->> "resources/public/img/map-pins" - clojure.java.io/file + (->> "public/img/map-pins" + io/resource + io/file file-seq (filter #(.isFile %)) (filter #(.matches grammar-matcher (.getFileName (.toPath %))))))) @@ -69,15 +70,20 @@ (defn get-data-file "Return JSON formatted data taken from the CSV file with the name `filename` - in the directory `resources/public/data`. TODO: There is a safe way to - access the content of the resource directory but I don't recall it just now." + in the directory `resources/public/data`." [filename] - (csv->json (io/reader (io/file (str "resources/public/data/" filename))))) + (-> (str "public/data/" filename) io/resource io/file io/reader csv->json)) (defn get-data + "Return JSON formatted data from the source implied by this `request`." [request] (ar/do-or-server-fail - (let [params (ac/massage-params request)] + ;; We're merging the parameters from the request with the key/value + ;; pairs already in the session, so that parame put into the session + ;; by calls to the home page can be used here. + (let [params (merge + (:session request) + (ac/massage-params request))] (cond (:docid params) (get-data-google (:docid params)) (:uri params) (get-data-uri (:uri params)) @@ -85,7 +91,8 @@ :else (get-data-file "data.csv"))) 200)) -(defroutes json-routes +(defroutes rest-routes (GET "/get-pin-image-names" request (get-pin-image-names request)) (POST "/get-pin-image-names" request (get-pin-image-names request)) - (GET "/get-data" request (get-data request))) + (GET "/get-data" request (get-data request)) + (POST "/get-data" request (get-data request))) diff --git a/src/cljs/geocsv/events.cljs b/src/cljs/geocsv/events.cljs index f620f71..f461a71 100644 --- a/src/cljs/geocsv/events.cljs +++ b/src/cljs/geocsv/events.cljs @@ -60,7 +60,7 @@ :fetch-data (fn [{db :db} _] (let [uri (assoc source-host - :path "/data/data.json")] + :path "/get-data")] (js/console.log (str "Fetching data: " uri)) diff --git a/src/cljs/geocsv/views/map.cljs b/src/cljs/geocsv/views/map.cljs index 181d37d..8d9d3b5 100644 --- a/src/cljs/geocsv/views/map.cljs +++ b/src/cljs/geocsv/views/map.cljs @@ -91,7 +91,7 @@ (defn map-render "Render the actual div containing the map." [] - [:div#map {:style {:height "500px"}}]) + [:div#map {:style {:height "1000px"}}]) (defn panel "A reagent class for the map object."