(ns gaza-stats.core "Functions to interrogate the [`gaza.onl`](https://gaza.onl/app.db) database." (:require [clojure.string :refer [capitalize]] [cljc.java-time.local-date :as ld] [com.hypirion.clj-xchart :refer [xy-chart]] [korma.core :refer [as-sql fields join select where]] [korma.db :refer [defdb sqlite3]]) (:import [java.util Date] [java.time LocalDate ZoneId ZoneOffset])) ;; create db (def db-spec "Database specification in a format which can be used with either JDBC or Korma." {:classname "org.sqlite.JDBC" :subprotocol "sqlite" :subname "resources/app.db"}) (defdb db ;; Korma-specific database handle. (sqlite3 db-spec)) (defmacro local-date? "Is this `x` an instance of `java.time.LocalDate`?" [x] `(instance? LocalDate ~x)) (defmacro normalise-date "More messing around with Java's bizarre mess of date/time classes. We want our default date representation to be `java.time.LocalDate`, except, of course, when we don't." [date default] `(cond (local-date? ~date) ~date (string? ~date) (ld/parse ~date) ;; this one should catch java.sql.Date as well, since it is a subclass (instance? Date ~date) (.toLocalDate (.atZone (.toInstant ~date) (ZoneId/systemDefault))) (nil? ~date) (ld/now) ;; if I'd written this as a fn, I'd just recurse,,, might be worth it. :else (if (local-date? ~default) ~default (ld/parse (str ~default))))) (defn total-donations-by-weeks "Return a list of pairs `[java.time.LocalDate date, float amount]` representing the total of donations made between `start-date` and `end-date`. Arguments may be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`. If `end-date` is not supplied, data for a single week will be returned." ([start-date] (let [sd (normalise-date start-date "2024-01-01")] (total-donations-by-weeks sd (ld/plus-days sd 7)))) ([start-date end-date] (let [sd (normalise-date start-date "2024-01-01") ed (normalise-date end-date (ld/now))] (loop [s sd i (ld/plus-days sd 7) r nil] (let [data (select 'campaign_donations (fields :amount) (where (and {:created_at [> s]} {:created_at [<= i]}))) amounts (map :amount data) r' (cons [s {:total (reduce + amounts) :n (count data) :max (apply max amounts) :min (apply min amounts)}] r)] (if (ld/is-before ed i) (reverse r') (recur i (ld/plus-days i 7) r'))))))) (defmacro local-date->date "Construct and return a new `java.util.Date` object from a `java.time.LocalDate` object. As usual, doing anything with Java time/date stuff is a complete bureaucratic nightmare." [x] `(Date/from (.toInstant (.atStartOfDay ~x) (ZoneOffset/ofHours 0)))) (defn chart-donations-by-weeks "Create and return an XY chart of data on field represented by `key` of donations made between `start-date` and `end-date`. Arguments may be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date key] (let [data (total-donations-by-weeks start-date end-date)] (xy-chart {(format "%s Donations" (capitalize (name key))) [(map #(local-date->date (first %)) data) (map #(:n (nth % 1)) data)]}))) (defn chart-total-donations-by-weeks "Create and return an XY chart of data on total donations made between `start-date` and `end-date`. Arguments may be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date] (chart-donations-by-weeks start-date end-date :total)) (defn chart-number-donations-by-weeks "Create and return an XY chart of data on number of donations made between `start-date` and `end-date`. Arguments may be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date] (chart-donations-by-weeks start-date end-date :n)) (defn enhance-account-with-donations "Take this `account_map`, which must at least have a valid value for the key `:campaign_url` and return a similar map including the key `:donations`, whose value is a float representing the total donations received by that campaign URL between the dates `sd` (exclusive) and `ed` (inclusive). Date arguments should be supplied as `java.time.LocalDate` objects." [account-map sd ed] (assoc account-map :donations (reduce + 0 (map :amount (select 'campaign_donations (fields :amount) (where (and {:campaign_url [= (:campaign_url account-map)]} {:created_at [> sd]} {:created_at [<= ed]}))))))) (defn active-accounts "Return, as an integer, the number of accounts believed to have been active on this `date`, which should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`" [date] (let [d (normalise-date date (ld/now))] (select 'accounts (fields :id :display_name :created_at :campaign_url) (where {:created_at [< d]})))) (defn accounts-with-donations-by-date-range "Return a list of maps, each representing one account from the `gaza.onl` database, with a key `:donations`, whose value is a float representing the total donations received by that account between the dates `start-date` (exclusive) and `end-date` (inclusive). Date arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date] (let [sd (normalise-date start-date "2024-01-01") ed (normalise-date end-date (ld/now))] (map #(enhance-account-with-donations % sd ed) (active-accounts sd)))) (defn chart-donations-by-accounts-in-date-range "Return a chart of donations received by individual accounts between `start-date` (exclusive) and `end-date` (inclusive). Arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`. Accounts are identified in the chart only by rank order." [start-date end-date] (let [data (accounts-with-donations-by-date-range start-date end-date)] (xy-chart {(format "Donations between\n%s and\n%s" start-date end-date) [(range (count data)) (sort (map :donations data))]}))) (defn chart-donations-as-percentage-in-date-range "Return a chart of donations received by individual accounts between `start-date` (exclusive) and `end-date` (inclusive). Arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`. Accounts are identified in the chart only by rank order." [start-date end-date] (let [data (accounts-with-donations-by-date-range start-date end-date) donations (sort (map :donations data)) total (reduce + donations) percentages (map #(* 100.0 (/ % total)) donations)] (xy-chart {(format "Donations as percent\nof total donations\nbetween %s and\n%s" start-date end-date) [(range (count data)) percentages]}))) (defn chart-accounts-with-donations-by-months-in-range "Return a chart with one line for each calendar month covering from `start-date` (inclusive) to `end-date` (inclusive). Arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`, showing donations as for `chart-donations-by-accounts-in-date-range`, q.v." [start-date end-date] (let [sd (normalise-date start-date "2024-01-01") ed (normalise-date end-date (ld/now))] (loop [s (.withDayOfMonth sd 1) i (.withDayOfMonth (.plusMonths sd 1) 1) m (sorted-map)] (let [data (accounts-with-donations-by-date-range s i) m' (assoc m (format "Donations between\n%s and\n%s" s i) [(range (count data)) (sort (map :donations data))])] (if (ld/is-before ed i) (xy-chart m') (recur i (.withDayOfMonth (.plusMonths i 1) 1) m')))))) ;; this does not work and I don't know why ;; (as-sql '(select 'campaign_donations ;; (fields :amount :campaign_url :created_at) ;; (where (and ;; {:campaign_url [= "https://www.chuffed.org/project/128261-photographer-from-gaza-rebuilding-life-dreams"]} ;; {:created_at [> "2024-04-01"]} ;; {:created_at [<= "2o24-04-30"]})))) (defn average-donations-received-by-month "Return data as a map keyed by the date of the first of each month between `start-date` and `end-date`; the values are maps with keys `:average`, `:count` and `:total`. Arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date] (let [sd (normalise-date start-date "2024-01-01") ed (normalise-date end-date (ld/now))] (loop [s (.withDayOfMonth sd 1) i (.withDayOfMonth (.plusMonths sd 1) 1) m (sorted-map)] (let [accs (accounts-with-donations-by-date-range s i) n (count accs) total (reduce + (map :donations accs)) av (if (zero? n) n (float (/ total n))) m' (assoc m s {:average av :count n :total total})] (if (ld/is-before ed i) m' (recur i (.withDayOfMonth (.plusMonths i 1) 1) m')))))) (defn chart-average-donations-received-by-month "Return a chart comparing the average donations received per account with the total number of active accounts per month between `start-date` and `end-date`. Arguments should be supplied as `java.time.LocalDate` objects, or as strings in the format `yyyy-mm-dd`." [start-date end-date] (let [data (average-donations-received-by-month start-date end-date)] (xy-chart {(format "Average donations\nper active account\nbetween\n%s and\n%s" start-date end-date) [(map #(local-date->date %) (keys data)) (map :average (vals data))] (format "Active accounts\nbetween\n%s and\n%s" start-date end-date) [(map #(local-date->date %) (keys data)) (map :count (vals data))]})))