Starting to get the project set up. Nothing is even nearly complete yet.

This commit is contained in:
simon 2016-10-13 14:25:54 +01:00
commit b6a24bc1ce
59 changed files with 7118 additions and 0 deletions

View file

@ -0,0 +1,10 @@
(ns youyesyet.config
(:require [cprop.core :refer [load-config]]
[cprop.source :as source]
[mount.core :refer [args defstate]]))
(defstate env :start (load-config
:merge
[(args)
(source/from-system-props)
(source/from-env)]))

View file

@ -0,0 +1,71 @@
(ns youyesyet.db.core
(:require
[cheshire.core :refer [generate-string parse-string]]
[clojure.java.jdbc :as jdbc]
[conman.core :as conman]
[youyesyet.config :refer [env]]
[mount.core :refer [defstate]])
(:import org.postgresql.util.PGobject
java.sql.Array
clojure.lang.IPersistentMap
clojure.lang.IPersistentVector
[java.sql
BatchUpdateException
Date
Timestamp
PreparedStatement]))
(defstate ^:dynamic *db*
:start (conman/connect! {:jdbc-url (env :database-url)})
:stop (conman/disconnect! *db*))
(conman/bind-connection *db* "sql/queries.sql")
(defn to-date [^java.sql.Date sql-date]
(-> sql-date (.getTime) (java.util.Date.)))
(extend-protocol jdbc/IResultSetReadColumn
Date
(result-set-read-column [v _ _] (to-date v))
Timestamp
(result-set-read-column [v _ _] (to-date v))
Array
(result-set-read-column [v _ _] (vec (.getArray v)))
PGobject
(result-set-read-column [pgobj _metadata _index]
(let [type (.getType pgobj)
value (.getValue pgobj)]
(case type
"json" (parse-string value true)
"jsonb" (parse-string value true)
"citext" (str value)
value))))
(extend-type java.util.Date
jdbc/ISQLParameter
(set-parameter [v ^PreparedStatement stmt ^long idx]
(.setTimestamp stmt idx (Timestamp. (.getTime v)))))
(defn to-pg-json [value]
(doto (PGobject.)
(.setType "jsonb")
(.setValue (generate-string value))))
(extend-type clojure.lang.IPersistentVector
jdbc/ISQLParameter
(set-parameter [v ^java.sql.PreparedStatement stmt ^long idx]
(let [conn (.getConnection stmt)
meta (.getParameterMetaData stmt)
type-name (.getParameterTypeName meta idx)]
(if-let [elem-type (when (= (first type-name) \_) (apply str (rest type-name)))]
(.setObject stmt idx (.createArrayOf conn elem-type (to-array v)))
(.setObject stmt idx (to-pg-json v))))))
(extend-protocol jdbc/ISQLValue
IPersistentMap
(sql-value [value] (to-pg-json value))
IPersistentVector
(sql-value [value] (to-pg-json value)))

View file

@ -0,0 +1,279 @@
(ns youyesyet.db.schema
(:require [clojure.java.jdbc :as sql]
[korma.core :as kc]
[youyesyet.db.core :as yyydb]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;
;;;; youyesyet.db.schema: database schema for youyesyet.
;;;;
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU General Public License
;;;; as published by the Free Software Foundation; either version 2
;;;; of the License, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU General Public License
;;;; along with this program; if not, write to the Free Software
;;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
;;;; USA.
;;;;
;;;; Copyright (C) 2016 Simon Brooke
;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn create-districts-table!
"Create a table to hold the electoral districts in which electors are registered.
Note that, as this app is being developed for the independence referendum in which
polling is across the whole of Scotland, this part of the design isn't fully thought
through; if later adapted to general or local elections, some breakdown or hierarchy
of polling districts into constituencies will be required."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:districts
[;; it may be necessary to have a serial abstract primary key but I suspect
;; polling districts already have numbers assigned by the Electoral Commission and
;; it would be sensible to use those. TODO: check.
[:id "integer not null primary key"]
[:name "varchar(64) not null"]
;; TODO: it would make sense to hold polygon data for polling districts so we can reflect
;; them on the map, but I haven't thought through how to do that yet.
])))
(kc/defentity district
(pk :id)
(table :districts)
(database yyydb/*db*)
(entity-fields :id :name))
(defn create-addresses-table!
"Create a table to hold the addresses at which electors are registered."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:addresses
[[:id "serial not null primary key"]
;; we do NOT want to hold multiple address records for the same household. When we receive
;; the electoral roll data the addresses are likely to be text fields inlined in the elector
;; record; in digesting the roll data we need to split these out and resolve them against existing
;; addresses in the table, creating a new address record only if there's no match.
[:address "varchar(256) not null unique"]
[:postcode "varchar(16)"]
[:phone "varchar(16)"]
;; the electoral district within which this address exists
[:district "integer references districts(id)"]
[:latitude :real]
[:longitude :real]])))
(kc/defentity address
(pk :id)
(table :addresses)
(database yyydb/*db*)
(entity-fields :id :address :postcode :phone :district :latitude :longitude))
(defn create-authority-table!
"Create a table to hold the oauth authorities against which we with authenticate canvassers."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:authority
[[:id "varchar(32) not null primary key"]
;; more stuff here when I understand more
])))
(kc/defentity authority
(pk :id)
(table :authorities)
(database yyydb/*db*)
(entity-fields :id :authority))
(defn create-electors-table!
"Create a table to hold electors data."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:electors
[[:rollno "integer primary key"]
[:name "varchar(64) not null"]
[:address "integer not null references addresses(id)" ]
[:phone "varchar(16)"]
;; we'll probably only capture email data on electors if they request a followup
;; on a particular issue by email.
[:email "varchar(128)"]])))
(kc/defentity elector
(pk :id)
(table :districts)
(database yyydb/*db*)
(entity-fields :id :name))
(defn create-canvassers-table!
"Create a table to hold data on canvassers (including authentication data)."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:canvassers
[[:username "varchar(32) not null primary key"]
[:fullname "varchar(64) not null"]
;; most canvassers will be electors, we should link them:
[:elector "integer references electors(rollno) on delete no action"]
;; but some canvassers may not be electors, so we need contact details separately:
[:address "integer not null references addresses(id)" ]
[:phone "varchar(16)"]
[:email "varchar(128)"]
;; with which authority do we authenticate this canvasser? I do not want to hold even
;; encrypted passwords locally
[:authority "varchar(32) not null references authority(id) on delete no action"]
;; true if the canvasser is authorised to use the app; else false. This allows us to
;; block canvassers we suspect of misbehaving.
[:authorised :boolean]])))
(defn create-visit-table!
"Create a table to record visits by canvassers to addresses (including virtual visits by telephone)."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:visits
[[:id "serial not null primary key"]
[:canvasser "varchar(32) references canvassers(username) not null"]
[:date "timestamp with timezone not null default now()"]])))
(defn create-option-table!
"Create a table to record options in the vote. This app is being created for the Independence
referendum, which will have just two options, 'Yes' and 'No', but it might later be adapted
for more general political canvassing."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:options
[[:option "varchar(32) not null primary key"
;; To do elections you probably need party and candidate and stuff here, but
;; for the referendum it's unnecessary.
]])))
(defn create-option-district-table!
"Create a table to link options to the districts in which they are relevant. This is extremely
simple for the referendum: both options are relevant to all districts. This table is essentially
'for later expansion'."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:optionsdistricts
[[:option "varchar(32) not null references options(option)"]
[:district "integer not null references districts(id)"]])))
(defn create-opinion-table!
"Create a table to record the opinion of an elector as solicited by a canvasser during a visit.
TODO: decide whether to insert a record in this table for 'don't knows'."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:opinions
[[:id "serial primary key"]
;; the elector who gave this opinion
[:elector "integer not null references electors(rollno)"]
;; the option the elector said they were planning to vote for
[:option "varchar(32) not null references options(option)"]
[:visit "integer not null references visits(id)"]])))
(defn create-issues-table!
"A table for issues we predict electors may raise on the doorstep, for which we may be
able to provide extra information or arrange for issue-specialists to phone and talk
to the elector."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:issues
[;; short name of this issue, e.g. 'currency', 'defence', 'pensions'
[:issue "varchar(32) not null primary key"]
;; URL of some brief material the canvasser can use on the doorstap
[:url "varchar(256)"]])))
(defn create-followup-method-table!
"Create a table to hold reference data on followup methods."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:followupmethod
[[:method "varchar(32) not null primary key"]])))
(defn create-issue-expertise-table!
"A table to record which canvassers have expertise in which issues, so that followup
requests can be directed to the right canvassers."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:issueexpertise
[[:expert "varchar(32) not null references canvasser(username)"]
[:issue "varchar(32) not null references issues(issue)"]
;; the method by which this expert can respond to electors on this issue
[:method "varchar 32 not null references followupmethod(method)"]])))
(defn create-followup-request-table!
"Create a table to record requests for followup contacts on particular issues."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:followuprequest
[[:id "serial primary key"]
[:elector "integer not null references electors(rollno)"]
[:visit "integer not null references visits(id)"]
[:issue "varchar(32) not null references issues(issue)"]
;; We probably need a followupmethod (telephone, email, postal) and, for telephone,
;; convenient times but I haven't thought through how to represent this or how
;; the user interface will work.
[:method "varchar(32) not null references followupmethod(method)"]])))
(defn create-followup-action-table!
"Create a table to record actions on followup requests. Record in this table are almost
certainly created through a desktop-style interface rather than through te app, so it's
reasonable that there should be narrative fields."
[]
(sql/db-do-commands
yyydb/*db*
(sql/create-table-ddl
:followupaction
[[:id "serial primary key"]
[:request "integer not null references followuprequest(id)"]
[:actor "varchar(32) not null references canvassers(username)"]
[:date "timestamp with timezone not null default now()"]
[:notes "text"]
;; true if this action closes the request
[:closed :boolean]])))

View file

@ -0,0 +1,47 @@
(ns youyesyet.handler
(:require [compojure.core :refer [routes wrap-routes]]
[youyesyet.layout :refer [error-page]]
[youyesyet.routes.home :refer [home-routes]]
[youyesyet.routes.oauth :refer [oauth-routes]]
[compojure.route :as route]
[youyesyet.env :refer [defaults]]
[mount.core :as mount]
[youyesyet.middleware :as middleware]
[clojure.tools.logging :as log]
[youyesyet.config :refer [env]]))
(mount/defstate init-app
:start ((or (:init defaults) identity))
:stop ((or (:stop defaults) identity)))
(defn init
"init will be called once when
app is deployed as a servlet on
an app server such as Tomcat
put any initialization code here"
[]
(doseq [component (:started (mount/start))]
(log/info component "started")))
(defn destroy
"destroy will be called when your application
shuts down, put any clean up code here"
[]
(doseq [component (:stopped (mount/stop))]
(log/info component "stopped"))
(shutdown-agents)
(log/info "youyesyet has shut down!"))
(def app-routes
(routes
(-> #'home-routes
(wrap-routes middleware/wrap-csrf)
(wrap-routes middleware/wrap-formats))
#'oauth-routes
(route/not-found
(:body
(error-page {:status 404
:title "page not found"})))))
(def app (middleware/wrap-base #'app-routes))

View file

@ -0,0 +1,39 @@
(ns youyesyet.layout
(:require [selmer.parser :as parser]
[selmer.filters :as filters]
[markdown.core :refer [md-to-html-string]]
[ring.util.http-response :refer [content-type ok]]
[ring.util.anti-forgery :refer [anti-forgery-field]]
[ring.middleware.anti-forgery :refer [*anti-forgery-token*]]))
(declare ^:dynamic *app-context*)
(parser/set-resource-path! (clojure.java.io/resource "templates"))
(parser/add-tag! :csrf-field (fn [_ _] (anti-forgery-field)))
(filters/add-filter! :markdown (fn [content] [:safe (md-to-html-string content)]))
(defn render
"renders the HTML template located relative to resources/templates"
[template & [params]]
(content-type
(ok
(parser/render-file
template
(assoc params
:page template
:csrf-token *anti-forgery-token*
:servlet-context *app-context*)))
"text/html; charset=utf-8"))
(defn error-page
"error-details should be a map containing the following keys:
:status - error status
:title - error title (optional)
:message - detailed error message (optional)
returns a response map with the error page as the body
and the status specified by the status key"
[error-details]
{:status (:status error-details)
:headers {"Content-Type" "text/html; charset=utf-8"}
:body (parser/render-file "error.html" error-details)})

View file

@ -0,0 +1,63 @@
(ns youyesyet.middleware
(:require [youyesyet.env :refer [defaults]]
[clojure.tools.logging :as log]
[youyesyet.layout :refer [*app-context* error-page]]
[ring.middleware.anti-forgery :refer [wrap-anti-forgery]]
[ring.middleware.webjars :refer [wrap-webjars]]
[ring.middleware.format :refer [wrap-restful-format]]
[youyesyet.config :refer [env]]
[ring-ttl-session.core :refer [ttl-memory-store]]
[ring.middleware.defaults :refer [site-defaults wrap-defaults]])
(:import [javax.servlet ServletContext]))
(defn wrap-context [handler]
(fn [request]
(binding [*app-context*
(if-let [context (:servlet-context request)]
;; If we're not inside a servlet environment
;; (for example when using mock requests), then
;; .getContextPath might not exist
(try (.getContextPath ^ServletContext context)
(catch IllegalArgumentException _ context))
;; if the context is not specified in the request
;; we check if one has been specified in the environment
;; instead
(:app-context env))]
(handler request))))
(defn wrap-internal-error [handler]
(fn [req]
(try
(handler req)
(catch Throwable t
(log/error t)
(error-page {:status 500
:title "Something very bad has happened!"
:message "We've dispatched a team of highly trained gnomes to take care of the problem."})))))
(defn wrap-csrf [handler]
(wrap-anti-forgery
handler
{:error-response
(error-page
{:status 403
:title "Invalid anti-forgery token"})}))
(defn wrap-formats [handler]
(let [wrapped (wrap-restful-format
handler
{:formats [:json-kw :transit-json :transit-msgpack]})]
(fn [request]
;; disable wrap-formats for websockets
;; since they're not compatible with this middleware
((if (:websocket? request) handler wrapped) request))))
(defn wrap-base [handler]
(-> ((:middleware defaults) handler)
wrap-webjars
(wrap-defaults
(-> site-defaults
(assoc-in [:security :anti-forgery] false)
(assoc-in [:session :store] (ttl-memory-store (* 60 30)))))
wrap-context
wrap-internal-error))

View file

@ -0,0 +1,35 @@
(ns youyesyet.oauth
(:require [youyesyet.config :refer [env]]
[oauth.client :as oauth]
[mount.core :refer [defstate]]
[clojure.tools.logging :as log]))
(defstate consumer
:start (oauth/make-consumer
(env :oauth-consumer-key)
(env :oauth-consumer-secret)
(env :request-token-uri)
(env :access-token-uri)
(env :authorize-uri)
:hmac-sha1))
(defn oauth-callback-uri
"Generates the oauth request callback URI"
[{:keys [headers]}]
(str (headers "x-forwarded-proto") "://" (headers "host") "/oauth/twitter-callback"))
(defn fetch-request-token
"Fetches a request token."
[request]
(let [callback-uri (oauth-callback-uri request)]
(log/info "Fetching request token using callback-uri" callback-uri)
(oauth/request-token consumer (oauth-callback-uri request))))
(defn fetch-access-token
[request_token]
(oauth/access-token consumer request_token (:oauth_verifier request_token)))
(defn auth-redirect-uri
"Gets the URI the user should be redirected to when authenticating."
[request-token]
(str (oauth/user-approval-uri consumer request-token)))

View file

@ -0,0 +1,15 @@
(ns youyesyet.routes.home
(:require [youyesyet.layout :as layout]
[youyesyet.db.core :as db-core]
[compojure.core :refer [defroutes GET]]
[ring.util.http-response :as response]
[clojure.java.io :as io]))
(defn home-page []
(layout/render "home.html"))
(defroutes home-routes
(GET "/" [] (home-page))
(GET "/docs" [] (-> (response/ok (-> "docs/docs.md" io/resource slurp))
(response/header "Content-Type" "text/plain; charset=utf-8"))))

View file

@ -0,0 +1,32 @@
(ns youyesyet.routes.oauth
(:require [compojure.core :refer [defroutes GET]]
[ring.util.http-response :refer [ok found]]
[clojure.java.io :as io]
[youyesyet.oauth :as oauth]
[clojure.tools.logging :as log]))
(defn oauth-init
"Initiates the Twitter OAuth"
[request]
(-> (oauth/fetch-request-token request)
:oauth_token
oauth/auth-redirect-uri
found))
(defn oauth-callback
"Handles the callback from Twitter."
[request_token {:keys [session]}]
; oauth request was denied by user
(if (:denied request_token)
(-> (found "/")
(assoc :flash {:denied true}))
; fetch the request token and do anything else you wanna do if not denied.
(let [{:keys [user_id screen_name]} (oauth/fetch-access-token request_token)]
(log/info "successfully authenticated as" user_id screen_name)
(-> (found "/")
(assoc :session
(assoc session :user-id user_id :screen-name screen_name))))))
(defroutes oauth-routes
(GET "/oauth/oauth-init" req (oauth-init req))
(GET "/oauth/oauth-callback" [& req_token :as req] (oauth-callback req_token req)))

View file

@ -0,0 +1,3 @@
(ns youyesyet.validation
(:require [bouncer.core :as b]
[bouncer.validators :as v]))

View file

@ -0,0 +1,20 @@
(ns youyesyet.ajax
(:require [ajax.core :as ajax]))
(defn local-uri? [{:keys [uri]}]
(not (re-find #"^\w+?://" uri)))
(defn default-headers [request]
(if (local-uri? request)
(-> request
(update :uri #(str js/context %))
(update :headers #(merge {"x-csrf-token" js/csrfToken} %)))
request))
(defn load-interceptors! []
(swap! ajax/default-interceptors
conj
(ajax/to-interceptor {:name "default headers"
:request default-headers})))

View file

@ -0,0 +1,95 @@
(ns youyesyet.core
(:require [reagent.core :as r]
[re-frame.core :as rf]
[secretary.core :as secretary]
[goog.events :as events]
[goog.history.EventType :as HistoryEventType]
[markdown.core :refer [md->html]]
[ajax.core :refer [GET POST]]
[youyesyet.ajax :refer [load-interceptors!]]
[youyesyet.handlers]
[youyesyet.subscriptions])
(:import goog.History))
(defn nav-link [uri title page collapsed?]
(let [selected-page (rf/subscribe [:page])]
[:li.nav-item
{:class (when (= page @selected-page) "active")}
[:a.nav-link
{:href uri
:on-click #(reset! collapsed? true)} title]]))
(defn navbar []
(r/with-let [collapsed? (r/atom true)]
[:nav.navbar.navbar-light.bg-faded
[:button.navbar-toggler.hidden-sm-up
{:on-click #(swap! collapsed? not)} "☰"]
[:div.collapse.navbar-toggleable-xs
(when-not @collapsed? {:class "in"})
[:a.navbar-brand {:href "#/"} "youyesyet"]
[:ul.nav.navbar-nav
[nav-link "#/" "Home" :home collapsed?]
[nav-link "#/about" "About" :about collapsed?]]]]))
(defn about-page []
[:div.container
[:div.row
[:div.col-md-12
"this is the story of youyesyet... work in progress"]]])
(defn home-page []
[:div.container
[:div.jumbotron
[:h1 "Welcome to youyesyet"]
[:p "Time to start building your site!"]
[:p [:a.btn.btn-primary.btn-lg {:href "http://luminusweb.net"} "Learn more »"]]]
(when-let [docs @(rf/subscribe [:docs])]
[:div.row
[:div.col-md-12
[:div {:dangerouslySetInnerHTML
{:__html (md->html docs)}}]]])])
(def pages
{:home #'home-page
:about #'about-page})
(defn page []
[:div
[navbar]
[(pages @(rf/subscribe [:page]))]])
;; -------------------------
;; Routes
(secretary/set-config! :prefix "#")
(secretary/defroute "/" []
(rf/dispatch [:set-active-page :home]))
(secretary/defroute "/about" []
(rf/dispatch [:set-active-page :about]))
;; -------------------------
;; History
;; must be called after routes have been defined
(defn hook-browser-navigation! []
(doto (History.)
(events/listen
HistoryEventType/NAVIGATE
(fn [event]
(secretary/dispatch! (.-token event))))
(.setEnabled true)))
;; -------------------------
;; Initialize app
(defn fetch-docs! []
(GET (str js/context "/docs") {:handler #(rf/dispatch [:set-docs %])}))
(defn mount-components []
(r/render [#'page] (.getElementById js/document "app")))
(defn init! []
(rf/dispatch-sync [:initialize-db])
(load-interceptors!)
(fetch-docs!)
(hook-browser-navigation!)
(mount-components))

View file

@ -0,0 +1,4 @@
(ns youyesyet.db)
(def default-db
{:page :home})

View file

@ -0,0 +1,18 @@
(ns youyesyet.handlers
(:require [youyesyet.db :as db]
[re-frame.core :refer [dispatch reg-event-db]]))
(reg-event-db
:initialize-db
(fn [_ _]
db/default-db))
(reg-event-db
:set-active-page
(fn [db [_ page]]
(assoc db :page page)))
(reg-event-db
:set-docs
(fn [db [_ docs]]
(assoc db :docs docs)))

View file

@ -0,0 +1,12 @@
(ns youyesyet.subscriptions
(:require [re-frame.core :refer [reg-sub]]))
(reg-sub
:page
(fn [db _]
(:page db)))
(reg-sub
:docs
(fn [db _]
(:docs db)))