simon/dog-and-duck
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Refactoring; internationalisation. Many tests failing.

Browse source
This commit is contained in:
Simon Brooke 2023-01-04 23:02:07 +00:00
parent e3f5078e9b
commit d26300f8c4
No known key found for this signature in database
GPG key ID: A7A4F18D1D4DF987
12 changed files with 263 additions and 215 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
project.clj
View file

@ -12,6 +12,7 @@
[com.taoensso/timbre "6.0.4"] [com.taoensso/timbre "6.0.4"]
[mvxcvi/clj-pgp "1.1.0"] [mvxcvi/clj-pgp "1.1.0"]
[org.bouncycastle/bcpkix-jdk18on "1.72"] [org.bouncycastle/bcpkix-jdk18on "1.72"]
[org.clojars.simon_brooke/internationalisation "1.0.4"]
[org.clojure/clojure "1.11.1"] [org.clojure/clojure "1.11.1"]
[org.clojure/data.json "2.4.0"] [org.clojure/data.json "2.4.0"]
[org.clojure/math.numeric-tower "0.0.5"] [org.clojure/math.numeric-tower "0.0.5"]

31
resources/i18n/en-GB.edn Normal file
View file

@ -0,0 +1,31 @@
;;; Copyright (C) Simon Brooke, 2023
;;; This program is free software; you can redistribute it and/or
;;; modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation; either version 2
;;; of the License, or (at your option) any later version.
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;; You should have received a copy of the GNU General Public License
;;; along with this program; if not, write to the Free Software
;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
;; Actual fault messages to which fault codes resolve: English language version.
{:expected-collection "A collection was expected, but was not found."
:id-not-https "Publicly facing content SHOULD use HTTPS URIs"
:id-not-uri "identifiers must be publicly dereferencable URIs"
:no-context "Section 3 of the ActivityPub specification states Implementers SHOULD include the ActivityPub context in their object definitions`."
:no-id-persistent "Persistent objects MUST have unique global identifiers."
:no-id-transient "The ActivityPub specification allows objects without `id` fields only if they are intentionally transient; even so it is preferred that the object should have an explicit null id."
:no-inbox "Actor objects MUST have an `inbox` property, whose value MUST be a reference to an ordered collection."
:no-items-collection "A collection expected to be simple had no items."
:no-outbox "Actor objects MUST have an `outbox` property, whose value MUST be a reference to an ordered collection."
:no-type "The ActivityPub specification states that the `type` field is optional, but it is hard to process objects with no known type."
:not-actor-type "The `type` value of the object was not a recognised actor type."
:not-valid-date-time "A date/time of format required for `xsd:dateTime` was expected but was not found."
:null-id-persistent "Persistent objects MUST have non-null identifiers."
:not-an-object "ActivityStreams object must be JSON objects."}

6
src/dog_and_duck/quack/picky.clj
View file

@ -31,15 +31,15 @@
paged-collection-faults paged-collection-faults
simple-collection-faults]] simple-collection-faults]]
[dog-and-duck.quack.picky.constants :refer [actor-types]] [dog-and-duck.quack.picky.constants :refer [actor-types]]
[dog-and-duck.quack.picky.objects :refer [coll-object-reference-or-fault
object-faults
object-reference-or-faults]]
[dog-and-duck.quack.picky.utils :refer [any-or-faults [dog-and-duck.quack.picky.utils :refer [any-or-faults
coll-object-reference-or-fault
concat-non-empty concat-non-empty
has-activity-type? has-activity-type?
has-actor-type? has-type? has-actor-type? has-type?
has-type-or-fault has-type-or-fault
make-fault-object make-fault-object
object-faults
object-reference-or-faults
string-or-fault]]) string-or-fault]])
(:import [java.net URI URISyntaxException])) (:import [java.net URI URISyntaxException]))

8
src/dog_and_duck/quack/picky/collections.clj
View file

@ -1,8 +1,8 @@
(ns dog-and-duck.quack.picky.collections (ns dog-and-duck.quack.picky.collections
(:require [dog-and-duck.quack.picky.utils :refer [concat-non-empty (:require [dog-and-duck.quack.picky.objects :refer [object-faults
cond-make-fault-object object-reference-or-faults]]
object-faults [dog-and-duck.quack.picky.utils :refer [concat-non-empty
object-reference-or-faults]])) cond-make-fault-object]]))
;;; Copyright (C) Simon Brooke, 2022 ;;; Copyright (C) Simon Brooke, 2022

7
src/dog_and_duck/quack/picky/constants.clj
View file

@ -22,13 +22,6 @@
literal string." literal string."
"https://www.w3.org/ns/activitystreams") "https://www.w3.org/ns/activitystreams")
(def ^:const xsd-date-time-pattern
"The pattern to which valid
[xsd:dateTime](https://www.w3.org/TR/xmlschema11-2/#dateTime) values conform.
TODO: this is failing on some of the published examples, so may be wrong."
"yyyy-MM-dd'T'HH:mm:ssX")
(def ^:const actor-types (def ^:const actor-types
"The set of types we will accept as actors. "The set of types we will accept as actors.

35
src/dog_and_duck/quack/picky/fault_messages.clj
View file

@ -1,35 +0,0 @@
(ns dog-and-duck.quack.picky.fault-messages
"Narrative values for fault reports of specific types, used by the picky
validator.")
;;; Copyright (C) Simon Brooke, 2022
;;; This program is free software; you can redistribute it and/or
;;; modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation; either version 2
;;; of the License, or (at your option) any later version.
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;; You should have received a copy of the GNU General Public License
;;; along with this program; if not, write to the Free Software
;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
(def messages
"Actual fault messages to which fault codes resolve."
{:expected-collection "A collection was expected, but was not found."
:id-not-https "Publicly facing content SHOULD use HTTPS URIs"
:id-not-uri "identifiers must be publicly dereferencable URIs"
:no-context "Section 3 of the ActivityPub specification states Implementers SHOULD include the ActivityPub context in their object definitions`."
:no-id-persistent "Persistent objects MUST have unique global identifiers."
:no-id-transient "The ActivityPub specification allows objects without `id` fields only if they are intentionally transient; even so it is preferred that the object should have an explicit null id."
:no-inbox "Actor objects MUST have an `inbox` property, whose value MUST be a reference to an ordered collection."
:no-items-collection "A collection expected to be simple had no items."
:no-outbox "Actor objects MUST have an `outbox` property, whose value MUST be a reference to an ordered collection."
:no-type "The ActivityPub specification states that the `type` field is optional, but it is hard to process objects with no known type."
:not-actor-type "The `type` value of the object was not a recognised actor type."
:null-id-persistent "Persistent objects MUST have non-null identifiers."
:not-an-object "ActivityStreams object must be JSON objects."})

140
src/dog_and_duck/quack/picky/objects.clj Normal file
View file

@ -0,0 +1,140 @@
(ns dog-and-duck.quack.picky.objects
(:require [clojure.data.json :as json]
[dog-and-duck.quack.picky.control-variables :refer [*reify-refs*]]
[dog-and-duck.quack.picky.time :refer [date-time-property-or-fault]]
[dog-and-duck.quack.picky.utils :refer [concat-non-empty
has-context?
has-type?
has-type-or-fault
make-fault-object
nil-if-empty]]
[taoensso.timbre :refer [warn]])
(:import [java.io FileNotFoundException]
[java.net URI URISyntaxException]))
(defn object-faults
"Return a list of faults found in object `x`, or `nil` if none are.
If `expected-type` is also passed, verify that `x` has `expected-type`.
`expected-type` may be passed as a string or as a set of strings. Detailed
verification of the particular features of types is not done here."
;; TODO: many more properties which are nor required, nevertheless have required
;; property TYPES as detailed in
;; https://www.w3.org/TR/activitystreams-vocabulary/#properties
;; if these properties are present, these types should be checked.
([x]
(nil-if-empty
(remove empty?
(list
(when-not (map? x)
(make-fault-object :critical :not-an-object))
(when-not
(has-context? x)
(make-fault-object :should :no-context))
(when-not (:type x)
(make-fault-object :minor :no-type))
(when-not (and (map? x) (contains? x :id))
(make-fault-object :minor :no-id-transient))
(date-time-property-or-fault x :endTime :must
:not-valid-date-time false)
(date-time-property-or-fault x :published :must
:not-valid-date-time false)
(date-time-property-or-fault x :startTime :must
:not-valid-date-time false)))))
([x expected-type]
(concat-non-empty
(object-faults x)
(when expected-type
(list
(has-type-or-fault x expected-type :critical :unexpected-type))))))
(def maybe-reify
"If `*reify-refs*` is `true`, return the object at this `target` URI.
Returns `nil` if
1. `*reify-refs*` is false;
2. the object was not found;
3. access to the object was not permitted.
Consequently, use with care."
(memoize
(fn [target]
(try (let [uri (URI. target)]
(when *reify-refs*
(json/read-str (slurp uri))))
(catch URISyntaxException _
(warn "Reification target" target "was not a valid URI.")
nil)
(catch FileNotFoundException _
(warn "Reification target" target "was not found.")
nil)))))
(defn maybe-reify-or-faults
"If `*reify-refs*` is `true`, runs basic checks on the object at this
`target` URI, if it is found, or a list containing a fault object with
this `severity` and `token` if it is not."
[value expected-type severity token]
(let [object (maybe-reify value)]
(cond object
(object-faults object expected-type)
*reify-refs* (list (make-fault-object severity token)))))
(defn object-reference-or-faults
"If this `value` is either
1. an object of `expected-type`;
2. a URI referencing an object of `expected-type`; or
3. a link object referencing an object of `expected-type`
and no faults are returned from validating the linked object, then return
`nil`; else return a sequence comprising a fault object with this `severity`
and `token`, prepended to the faults returned.
As with `has-type-or-fault` (q.v.), `expected-type` may be passed as a
string, as a set of strings, or `nil` (indicating the type of the
referenced object should not be checked).
**NOTE THAT** if `*reify-refs*` is `false`, referenced objects will not
actually be checked."
[value expected-type severity token]
(let [faults (cond
(string? value) (maybe-reify-or-faults value severity token expected-type)
(map? value) (if (has-type? value "Link")
(cond
;; if we were looking for a link and we've
;; found a link, that's OK.
(= expected-type "Link") nil
(and (set? expected-type) (expected-type "Link")) nil
(nil? expected-type) nil
:else
(object-reference-or-faults
(:href value) expected-type severity token))
(object-faults value expected-type))
:else (throw
(ex-info
"Argument `value` was not an object or a link to an object"
{:arguments {:value value}
:expected-type expected-type
:severity severity
:token token})))]
(when faults (cons (make-fault-object severity token) faults))))
(defn coll-object-reference-or-fault
"As object-reference-or-fault, except `value` argument may also be a list of
objects and/or object references."
[value expected-type severity token]
(cond
(map? value) (object-reference-or-faults value expected-type severity token)
(coll? value) (concat-non-empty
(map
#(object-reference-or-faults
% expected-type severity token)
value))
:else (throw
(ex-info
"Argument `value` was not an object, a link to an object, nor a list of these."
{:arguments {:value value}
:expected-type expected-type
:severity severity
:token token}))))

47
src/dog_and_duck/quack/picky/time.clj Normal file
View file

@ -0,0 +1,47 @@
(ns dog-and-duck.quack.picky.time
"Time, gentleman, please! Recognising and validating date time values."
(:require [dog-and-duck.quack.picky.utils :refer [cond-make-fault-object
make-fault-object]]
[scot.weft.i18n.core :refer [get-message]]
[taoensso.timbre :refer [warn]])
(:import [java.time LocalDateTime]
[java.time.format DateTimeFormatter DateTimeParseException]))
;;; Copyright (C) Simon Brooke, 2023
;;; This program is free software; you can redistribute it and/or
;;; modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation; either version 2
;;; of the License, or (at your option) any later version.
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;; You should have received a copy of the GNU General Public License
;;; along with this program; if not, write to the Free Software
;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
(defn xsd-date-time?
"Return `true` if `value` matches the pattern for an
[xsd:dateTime](https://www.w3.org/TR/xmlschema11-2/#dateTime), else `false`"
[^String value]
(try
(if (LocalDateTime/from (.parse DateTimeFormatter/ISO_DATE_TIME value)) true false)
(catch DateTimeParseException _
(warn (get-message :bad-date-time) ":" value)
false)))
(defn date-time-property-or-fault
"If the value of this `property` of object `x` is a valid xsd:dateTime
value, return a fault object with this `token` and `severity`.
If `required?` is false and there is no such property, no fault will be
returned."
[x property severity token required?]
(let [value (property x)]
(if (and required? (not (x property)))
(make-fault-object severity token)
(cond-make-fault-object
(and value (xsd-date-time? value)) severity token))))

166
src/dog_and_duck/quack/picky/utils.clj
View file

@ -1,22 +1,16 @@
(ns dog-and-duck.quack.picky.utils (ns dog-and-duck.quack.picky.utils
"Utility functions supporting the picky validator" "Utility functions supporting the picky validator"
(:require [clojure.data.json :as json] (:require [clojure.set :refer [intersection]]
[java-time.api :as jt]
[clojure.set :refer [intersection]]
[dog-and-duck.quack.picky.constants :refer [activitystreams-context-uri [dog-and-duck.quack.picky.constants :refer [activitystreams-context-uri
actor-types actor-types
context-key severity-filters context-key severity-filters
validation-fault-context-uri validation-fault-context-uri
verb-types verb-types]]
xsd-date-time-pattern]]
[dog-and-duck.quack.picky.control-variables :refer [*reify-refs*]]
[dog-and-duck.quack.picky.fault-messages :refer [messages]]
[dog-and-duck.utils.process :refer [get-hostname get-pid]] [dog-and-duck.utils.process :refer [get-hostname get-pid]]
[scot.weft.i18n.core :refer [get-message]]
[taoensso.timbre :as log :refer [warn]]) [taoensso.timbre :as log :refer [warn]])
(:import [java.io FileNotFoundException] (:import [java.net URI URISyntaxException]))
[java.net URI URISyntaxException]
[java.time.format DateTimeParseException]))
;;; Copyright (C) Simon Brooke, 2022 ;;; Copyright (C) Simon Brooke, 2022
@ -156,7 +150,7 @@
:type "Fault" :type "Fault"
:severity severity :severity severity
:fault fault :fault fault
:narrative (or (messages fault) :narrative (or (get-message fault)
(do (do
(warn "No narrative provided for fault token " fault) (warn "No narrative provided for fault token " fault)
(str fault))))) (str fault)))))
@ -231,153 +225,3 @@
([value severity token pattern] ([value severity token pattern]
(when not (and (string? value) (re-matches pattern value)) (when not (and (string? value) (re-matches pattern value))
(make-fault-object severity token)))) (make-fault-object severity token))))
(defn xsd-date-time?
"Return `true` if `value` matches the pattern for an
[xsd:dateTime](https://www.w3.org/TR/xmlschema11-2/#dateTime), else `false`"
[^String value]
(try
(if (jt/local-date-time xsd-date-time-pattern value) true false)
(catch DateTimeParseException _
(log/warn "Not a recognised xsd:dateTime: " value)
false)))
(defn date-time-property-or-fault
"If the value of this `property` of object `x` is a valid xsd:dateTime
value, return a fault object with this `token` and `severity`.
If `required?` is false and there is no such property, no fault will be
returned."
[x property severity token required?]
(let [value (property x)]
(if (and required? (not (x property)))
(make-fault-object severity token)
(cond-make-fault-object
(and value (xsd-date-time? value)) severity token))))
(defn object-faults
"Return a list of faults found in object `x`, or `nil` if none are.
If `expected-type` is also passed, verify that `x` has `expected-type`.
`expected-type` may be passed as a string or as a set of strings. Detailed
verification of the particular features of types is not done here."
;; TODO: many more properties which are nor required, nevertheless have required
;; property TYPES as detailed in
;; https://www.w3.org/TR/activitystreams-vocabulary/#properties
;; if these properties are present, these types should be checked.
([x]
(nil-if-empty
(remove empty?
(list
(when-not (map? x)
(make-fault-object :critical :not-an-object))
(when-not
(has-context? x)
(make-fault-object :should :no-context))
(when-not (:type x)
(make-fault-object :minor :no-type))
(when-not (and (map? x) (contains? x :id))
(make-fault-object :minor :no-id-transient))
(date-time-property-or-fault x :endTime :must
:not-valid-date-time false)
(date-time-property-or-fault x :published :must
:not-valid-date-time false)
(date-time-property-or-fault x :startTime :must
:not-valid-date-time false)))))
([x expected-type]
(concat-non-empty
(object-faults x)
(when expected-type
(list
(has-type-or-fault x expected-type :critical :unexpected-type))))))
(def maybe-reify
"If `*reify-refs*` is `true`, return the object at this `target` URI.
Returns `nil` if
1. `*reify-refs*` is false;
2. the object was not found;
3. access to the object was not permitted.
Consequently, use with care."
(memoize
(fn [target]
(try (let [uri (URI. target)]
(when *reify-refs*
(json/read-str (slurp uri))))
(catch URISyntaxException _
(log/warn "Reification target" target "was not a valid URI.")
nil)
(catch FileNotFoundException _
(log/warn "Reification target" target "was not found.")
nil)))))
(defn maybe-reify-or-faults
"If `*reify-refs*` is `true`, runs basic checks on the object at this
`target` URI, if it is found, or a list containing a fault object with
this `severity` and `token` if it is not."
[value expected-type severity token]
(let [object (maybe-reify value)]
(cond object
(object-faults object expected-type)
*reify-refs* (list (make-fault-object severity token)))))
(defn object-reference-or-faults
"If this `value` is either
1. an object of `expected-type`;
2. a URI referencing an object of `expected-type`; or
3. a link object referencing an object of `expected-type`
and no faults are returned from validating the linked object, then return
`nil`; else return a sequence comprising a fault object with this `severity`
and `token`, prepended to the faults returned.
As with `has-type-or-fault` (q.v.), `expected-type` may be passed as a
string, as a set of strings, or `nil` (indicating the type of the
referenced object should not be checked).
**NOTE THAT** if `*reify-refs*` is `false`, referenced objects will not
actually be checked."
[value expected-type severity token]
(let [faults (cond
(string? value) (maybe-reify-or-faults value severity token expected-type)
(map? value) (if (has-type? value "Link")
(cond
;; if we were looking for a link and we've
;; found a link, that's OK.
(= expected-type "Link") nil
(and (set? expected-type) (expected-type "Link")) nil
(nil? expected-type) nil
:else
(object-reference-or-faults
(:href value) expected-type severity token))
(object-faults value expected-type))
:else (throw
(ex-info
"Argument `value` was not an object or a link to an object"
{:arguments {:value value}
:expected-type expected-type
:severity severity
:token token})))]
(when faults (cons (make-fault-object severity token) faults))))
(defn coll-object-reference-or-fault
"As object-reference-or-fault, except `value` argument may also be a list of
objects and/or object references."
[value expected-type severity token]
(cond
(map? value) (object-reference-or-faults value expected-type severity token)
(coll? value) (concat-non-empty
(map
#(object-reference-or-faults
% expected-type severity token)
value))
:else (throw
(ex-info
"Argument `value` was not an object, a link to an object, nor a list of these."
{:arguments {:value value}
:expected-type expected-type
:severity severity
:token token}))))

3
src/dog_and_duck/quack/quack.clj
View file

@ -17,7 +17,8 @@
link-faults link-faults
persistent-object-faults]] persistent-object-faults]]
[dog-and-duck.quack.picky.control-variables :refer [*reject-severity*]] [dog-and-duck.quack.picky.control-variables :refer [*reject-severity*]]
[dog-and-duck.quack.picky.utils :refer [filter-severity object-faults]]) [dog-and-duck.quack.picky.objects :refer [object-faults]]
[dog-and-duck.quack.picky.utils :refer [filter-severity]])
(:import [java.net URI URISyntaxException])) (:import [java.net URI URISyntaxException]))

26
test/dog_and_duck/quack/picky/time_test.clj Normal file
View file

@ -0,0 +1,26 @@
(ns dog-and-duck.quack.picky.time-test
(:require [clojure.test :refer [deftest is testing]]
[dog-and-duck.quack.picky.time :refer
[date-time-property-or-fault xsd-date-time?]]))
;;; Copyright (C) Simon Brooke, 2023
;;; This program is free software; you can redistribute it and/or
;;; modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation; either version 2
;;; of the License, or (at your option) any later version.
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;; GNU General Public License for more details.
;;; You should have received a copy of the GNU General Public License
;;; along with this program; if not, write to the Free Software
;;; Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
(deftest date-time-test
(testing "xsd-date-time?"
(let [expected true
actual (xsd-date-time? "2002-05-30T09:00:00")]
(is (= actual expected)))))

4
test/dog_and_duck/quack/picky_test.clj
View file

@ -2,8 +2,8 @@
(:require [clojure.test :refer [deftest is testing]] (:require [clojure.test :refer [deftest is testing]]
[dog-and-duck.quack.picky.constants :refer [dog-and-duck.quack.picky.constants :refer
[activitystreams-context-uri]] [activitystreams-context-uri]]
[dog-and-duck.quack.picky.utils :refer [dog-and-duck.quack.picky.objects :refer [object-faults]]
[filter-severity object-faults]] [dog-and-duck.quack.picky.utils :refer [filter-severity]]
[dog-and-duck.quack.picky :refer [dog-and-duck.quack.picky :refer
[collection-faults persistent-object-faults]] [collection-faults persistent-object-faults]]
[dog-and-duck.scratch.parser :refer [clean]])) [dog-and-duck.scratch.parser :refer [clean]]))