simon/IReadIt
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Test data, and a couple of unit tests.

Browse source
This commit is contained in:
Simon Brooke 2019-02-19 16:42:49 +00:00
parent 71d7fd151d
commit 90e42c691e
8 changed files with 705 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
env/test/resources/data/extended_tweet_280_characters.edn vendored Normal file
View file

@ -0,0 +1,113 @@
{:quote_count 0,
:in_reply_to_screen_name nil,
:is_quote_status false,
:matching_rules [{:tag nil}],
:coordinates nil,
:filter_level "low",
:in_reply_to_status_id_str nil,
:place
{:name "Boulder",
:place_type "city",
:country_code "US",
:id "fd70c22040963ac7",
:url
"https://api.twitter.com/1.1/geo/id/fd70c22040963ac7.json",
:bounding_box
{:type "Polygon",
:coordinates
[[[-105.301776 39.953552]
[-105.301776 40.094411]
[-105.183597 40.094411]
[-105.183597 39.953552]]]},
:full_name "Boulder, CO",
:attributes {},
:country "United States"},
:geo nil,
:in_reply_to_status_id nil,
:extended_tweet
{:full_text
"So this is my first experimental Tweet with 280 characters, enabling me to express all my hopes and dreams for the future of the Twitter platform, while also ignoring the voices in my head about poorly written parsers exploding around the globe... #AddingText4Exactly280Characters",
:display_text_range [0 280],
:entities
{:hashtags
[{:text "AddingText4Exactly280Characters",
:indices [248 280]}],
:urls [],
:user_mentions [],
:symbols []}},
:entities
{:hashtags [],
:urls
[{:url "https://t.co/Nz6dQf3rIr",
:expanded_url
"https://twitter.com/i/web/status/912785533910306817",
:display_url "twitter.com/i/web/status/9…",
:indices [116 139]}],
:user_mentions [],
:symbols []},
:source "Twitter Web Client",
:lang "en",
:in_reply_to_user_id_str nil,
:id 912785533910306800,
:contributors nil,
:truncated true,
:retweeted false,
:in_reply_to_user_id nil,
:id_str "912785533910306817",
:favorited false,
:user
{:description
"family, travel, music, snow, urban farming, photography, coding, weather, hydrology, early-warning systems. From Minnesota, live in Colorado.",
:profile_link_color "ABB8C2",
:profile_sidebar_border_color "FFFFFF",
:profile_image_url
"http://pbs.twimg.com/profile_images/697445972402540546/2CYK3PWX_normal.jpg",
:profile_use_background_image true,
:default_profile false,
:profile_background_image_url
"http://pbs.twimg.com/profile_background_images/846867852/0962a373696e2080d26659b920880fc8.jpeg",
:is_translator false,
:profile_text_color "333333",
:profile_banner_url
"https://pbs.twimg.com/profile_banners/17200003/1399004533",
:name "Jim ❄️☃🌎",
:profile_background_image_url_https
"https://pbs.twimg.com/profile_background_images/846867852/0962a373696e2080d26659b920880fc8.jpeg",
:favourites_count 2094,
:screen_name "snowman",
:listed_count 39,
:profile_image_url_https
"https://pbs.twimg.com/profile_images/697445972402540546/2CYK3PWX_normal.jpg",
:statuses_count 1637,
:contributors_enabled false,
:following nil,
:lang "en",
:utc_offset -21600,
:notifications nil,
:default_profile_image false,
:profile_background_color "C0DEED",
:id 17200003,
:follow_request_sent nil,
:url nil,
:translator_type "regular",
:time_zone "Mountain Time (US & Canada)",
:profile_sidebar_fill_color "DDEEF6",
:protected false,
:profile_background_tile true,
:id_str "17200003",
:geo_enabled true,
:location "MN ⇨ all over ⇨ CO",
:followers_count 751,
:friends_count 457,
:verified false,
:created_at
"Wed Nov 05 23:06:37 +0000 2008"},
:reply_count 4,
:retweet_count 0,
:favorite_count 2,
:created_at "Tue Sep 26 21:06:44 +0000 2017",
:text
"So this is my first experimental Tweet with 280 characters, enabling me to express all my hopes and dreams for the… https://t.co/Nz6dQf3rIr"
}

156
env/test/resources/data/extended_tweet_280_characters.json vendored Normal file
View file

@ -0,0 +1,156 @@
{
"created_at": "Tue Sep 26 21:06:44 +0000 2017",
"id": 912785533910306817,
"id_str": "912785533910306817",
"text": "So this is my first experimental Tweet with 280 characters, enabling me to express all my hopes and dreams for the… https://t.co/Nz6dQf3rIr",
"source": "Twitter Web Client",
"truncated": true,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 17200003,
"id_str": "17200003",
"name": "Jim ❄️☃🌎",
"screen_name": "snowman",
"location": "MN ⇨ all over ⇨ CO",
"url": null,
"description": "family, travel, music, snow, urban farming, photography, coding, weather, hydrology, early-warning systems. From Minnesota, live in Colorado.",
"translator_type": "regular",
"protected": false,
"verified": false,
"followers_count": 751,
"friends_count": 457,
"listed_count": 39,
"favourites_count": 2094,
"statuses_count": 1637,
"created_at": "Wed Nov 05 23:06:37 +0000 2008",
"utc_offset": -21600,
"time_zone": "Mountain Time (US & Canada)",
"geo_enabled": true,
"lang": "en",
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "C0DEED",
"profile_background_image_url": "http://pbs.twimg.com/profile_background_images/846867852/0962a373696e2080d26659b920880fc8.jpeg",
"profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/846867852/0962a373696e2080d26659b920880fc8.jpeg",
"profile_background_tile": true,
"profile_link_color": "ABB8C2",
"profile_sidebar_border_color": "FFFFFF",
"profile_sidebar_fill_color": "DDEEF6",
"profile_text_color": "333333",
"profile_use_background_image": true,
"profile_image_url": "http://pbs.twimg.com/profile_images/697445972402540546/2CYK3PWX_normal.jpg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/697445972402540546/2CYK3PWX_normal.jpg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/17200003/1399004533",
"default_profile": false,
"default_profile_image": false,
"following": null,
"follow_request_sent": null,
"notifications": null
},
"geo": null,
"coordinates": null,
"place": {
"id": "fd70c22040963ac7",
"url": "https://api.twitter.com/1.1/geo/id/fd70c22040963ac7.json",
"place_type": "city",
"name": "Boulder",
"full_name": "Boulder, CO",
"country_code": "US",
"country": "United States",
"bounding_box": {
"type": "Polygon",
"coordinates": [
[
[
-105.301776,
39.953552
],
[
-105.301776,
40.094411
],
[
-105.183597,
40.094411
],
[
-105.183597,
39.953552
]
]
]
},
"attributes": {
}
},
"contributors": null,
"is_quote_status": false,
"extended_tweet": {
"full_text": "So this is my first experimental Tweet with 280 characters, enabling me to express all my hopes and dreams for the future of the Twitter platform, while also ignoring the voices in my head about poorly written parsers exploding around the globe... #AddingText4Exactly280Characters",
"display_text_range": [
0,
280
],
"entities": {
"hashtags": [
{
"text": "AddingText4Exactly280Characters",
"indices": [
248,
280
]
}
],
"urls": [
],
"user_mentions": [
],
"symbols": [
]
}
},
"quote_count": 0,
"reply_count": 4,
"retweet_count": 0,
"favorite_count": 2,
"entities": {
"hashtags": [
],
"urls": [
{
"url": "https://t.co/Nz6dQf3rIr",
"expanded_url": "https://twitter.com/i/web/status/912785533910306817",
"display_url": "twitter.com/i/web/status/9…",
"indices": [
116,
139
]
}
],
"user_mentions": [
],
"symbols": [
]
},
"favorited": false,
"retweeted": false,
"filter_level": "low",
"lang": "en",
"matching_rules": [
{
"tag": null
}
]
}

147
env/test/resources/data/original_format_image_alt_text.edn vendored Normal file
View file

@ -0,0 +1,147 @@
{:quote_count 0,
:in_reply_to_screen_name nil,
:is_quote_status false,
:matching_rules
[{:tag nil, :id 794211184496951300}],
:coordinates nil,
:filter_level "low",
:in_reply_to_status_id_str nil,
:place nil,
:timestamp_ms "1485361639674",
:possibly_sensitive false,
:geo nil,
:in_reply_to_status_id nil,
:entities
{:hashtags [],
:urls [],
:user_mentions [],
:symbols [],
:media
[{:description "Idiot",
:sizes
{:small {:w 680, :h 383, :resize "fit"},
:medium {:w 800, :h 450, :resize "fit"},
:thumb {:w 150, :h 150, :resize "crop"},
:large {:w 800, :h 450, :resize "fit"}},
:media_url_https
"https://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg",
:type "photo",
:media_url
"http://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg",
:id 824292386197020700,
:expanded_url
"https://twitter.com/TestMcTest_/status/824292577188773888/photo/1",
:url "https://t.co/Y286t9R6Bi",
:display_url "pic.twitter.com/Y286t9R6Bi",
:indices [23 46],
:id_str "824292386197020672"}]},
:source
"<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
:lang "en",
:in_reply_to_user_id_str nil,
:id 824292577188773900,
:contributors nil,
:extended_entities
{:media
[{:description "Idiot",
:sizes
{:small {:w 680, :h 383, :resize "fit"},
:medium {:w 800, :h 450, :resize "fit"},
:thumb {:w 150, :h 150, :resize "crop"},
:large {:w 800, :h 450, :resize "fit"}},
:media_url_https
"https://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg",
:type "photo",
:media_url
"http://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg",
:id 824292386197020700,
:expanded_url
"https://twitter.com/TestMcTest_/status/824292577188773888/photo/1",
:url "https://t.co/Y286t9R6Bi",
:display_url "pic.twitter.com/Y286t9R6Bi",
:indices [23 46],
:id_str "824292386197020672"}
{:description "Dog",
:sizes
{:medium {:w 720, :h 960, :resize "fit"},
:large {:w 720, :h 960, :resize "fit"},
:thumb {:w 150, :h 150, :resize "crop"},
:small {:w 510, :h 680, :resize "fit"}},
:media_url_https
"https://pbs.twimg.com/media/C3B5nR3VIAAKJPn.jpg",
:type "photo",
:media_url
"http://pbs.twimg.com/media/C3B5nR3VIAAKJPn.jpg",
:id 824292447526133800,
:expanded_url
"https://twitter.com/TestMcTest_/status/824292577188773888/photo/1",
:url "https://t.co/Y286t9R6Bi",
:display_url "pic.twitter.com/Y286t9R6Bi",
:indices [23 46],
:id_str "824292447526133760"}]},
:display_text_range [0 22],
:truncated false,
:retweeted false,
:in_reply_to_user_id nil,
:id_str "824292577188773888",
:favorited false,
:user
{:description "http://www.gnip.com",
:profile_link_color "1DA1F2",
:profile_sidebar_border_color "C0DEED",
:profile_image_url
"http://abs.twimg.com/sticky/default_profile_images/default_profile_3_normal.png",
:profile_use_background_image true,
:default_profile true,
:profile_background_image_url "",
:is_translator false,
:profile_text_color "333333",
:name "Test McTest",
:profile_background_image_url_https "",
:favourites_count 1,
:screen_name "TestMcTest_",
:listed_count 0,
:profile_image_url_https
"https://abs.twimg.com/sticky/default_profile_images/default_profile_3_normal.png",
:statuses_count 3,
:contributors_enabled false,
:following nil,
:lang "en",
:utc_offset nil,
:notifications nil,
:default_profile_image true,
:profile_background_color "F5F8FA",
:id 750795692558483500,
:follow_request_sent nil,
:derived
{:locations
[{:country "United States",
:country_code "US",
:locality "Boulder",
:region "Colorado",
:sub_region "Boulder County",
:full_name
"Boulder, Colorado, United States",
:geo
{:coordinates [-105.27055 40.01499],
:type "point"}}]},
:url "http://www.gnip.com",
:translator_type "none",
:time_zone nil,
:profile_sidebar_fill_color "DDEEF6",
:protected false,
:profile_background_tile false,
:id_str "750795692558483456",
:geo_enabled false,
:location "Boulder, CO",
:followers_count 1,
:friends_count 1,
:verified false,
:created_at
"Wed Jul 06 20:56:57 +0000 2016"},
:reply_count 0,
:retweet_count 0,
:favorite_count 0,
:created_at "Wed Jan 25 16:27:19 +0000 2017",
:text
"Image Alt-Text Example https://t.co/Y286t9R6Bi"}

228
env/test/resources/data/original_format_image_alt_text.json vendored Normal file
View file

@ -0,0 +1,228 @@
{
"created_at":"Wed Jan 25 16:27:19 +0000 2017",
"id":824292577188773888,
"id_str":"824292577188773888",
"text":"Image Alt-Text Example https:\/\/t.co\/Y286t9R6Bi",
"display_text_range":[
0,
22
],
"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e",
"truncated":false,
"in_reply_to_status_id":null,
"in_reply_to_status_id_str":null,
"in_reply_to_user_id":null,
"in_reply_to_user_id_str":null,
"in_reply_to_screen_name":null,
"user":{
"id":750795692558483456,
"id_str":"750795692558483456",
"name":"Test McTest",
"screen_name":"TestMcTest_",
"location":"Boulder, CO",
"url":"http:\/\/www.gnip.com",
"description":"http:\/\/www.gnip.com",
"translator_type":"none",
"derived":{
"locations":[
{
"country":"United States",
"country_code":"US",
"locality":"Boulder",
"region":"Colorado",
"sub_region":"Boulder County",
"full_name":"Boulder, Colorado, United States",
"geo":{
"coordinates":[
-105.27055,
40.01499
],
"type":"point"
}
}
]
},
"protected":false,
"verified":false,
"followers_count":1,
"friends_count":1,
"listed_count":0,
"favourites_count":1,
"statuses_count":3,
"created_at":"Wed Jul 06 20:56:57 +0000 2016",
"utc_offset":null,
"time_zone":null,
"geo_enabled":false,
"lang":"en",
"contributors_enabled":false,
"is_translator":false,
"profile_background_color":"F5F8FA",
"profile_background_image_url":"",
"profile_background_image_url_https":"",
"profile_background_tile":false,
"profile_link_color":"1DA1F2",
"profile_sidebar_border_color":"C0DEED",
"profile_sidebar_fill_color":"DDEEF6",
"profile_text_color":"333333",
"profile_use_background_image":true,
"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_3_normal.png",
"profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_3_normal.png",
"default_profile":true,
"default_profile_image":true,
"following":null,
"follow_request_sent":null,
"notifications":null
},
"geo":null,
"coordinates":null,
"place":null,
"contributors":null,
"is_quote_status":false,
"quote_count":0,
"reply_count":0,
"retweet_count":0,
"favorite_count":0,
"entities":{
"hashtags":[
],
"urls":[
],
"user_mentions":[
],
"symbols":[
],
"media":[
{
"id":824292386197020672,
"id_str":"824292386197020672",
"indices":[
23,
46
],
"description":"Idiot",
"media_url":"http:\/\/pbs.twimg.com\/media\/C3B5jtZVIAAtCue.jpg",
"media_url_https":"https:\/\/pbs.twimg.com\/media\/C3B5jtZVIAAtCue.jpg",
"url":"https:\/\/t.co\/Y286t9R6Bi",
"display_url":"pic.twitter.com\/Y286t9R6Bi",
"expanded_url":"https:\/\/twitter.com\/TestMcTest_\/status\/824292577188773888\/photo\/1",
"type":"photo",
"sizes":{
"small":{
"w":680,
"h":383,
"resize":"fit"
},
"medium":{
"w":800,
"h":450,
"resize":"fit"
},
"thumb":{
"w":150,
"h":150,
"resize":"crop"
},
"large":{
"w":800,
"h":450,
"resize":"fit"
}
}
}
]
},
"extended_entities":{
"media":[
{
"id":824292386197020672,
"id_str":"824292386197020672",
"indices":[
23,
46
],
"description":"Idiot",
"media_url":"http:\/\/pbs.twimg.com\/media\/C3B5jtZVIAAtCue.jpg",
"media_url_https":"https:\/\/pbs.twimg.com\/media\/C3B5jtZVIAAtCue.jpg",
"url":"https:\/\/t.co\/Y286t9R6Bi",
"display_url":"pic.twitter.com\/Y286t9R6Bi",
"expanded_url":"https:\/\/twitter.com\/TestMcTest_\/status\/824292577188773888\/photo\/1",
"type":"photo",
"sizes":{
"small":{
"w":680,
"h":383,
"resize":"fit"
},
"medium":{
"w":800,
"h":450,
"resize":"fit"
},
"thumb":{
"w":150,
"h":150,
"resize":"crop"
},
"large":{
"w":800,
"h":450,
"resize":"fit"
}
}
},
{
"id":824292447526133760,
"id_str":"824292447526133760",
"indices":[
23,
46
],
"description":"Dog",
"media_url":"http:\/\/pbs.twimg.com\/media\/C3B5nR3VIAAKJPn.jpg",
"media_url_https":"https:\/\/pbs.twimg.com\/media\/C3B5nR3VIAAKJPn.jpg",
"url":"https:\/\/t.co\/Y286t9R6Bi",
"display_url":"pic.twitter.com\/Y286t9R6Bi",
"expanded_url":"https:\/\/twitter.com\/TestMcTest_\/status\/824292577188773888\/photo\/1",
"type":"photo",
"sizes":{
"medium":{
"w":720,
"h":960,
"resize":"fit"
},
"large":{
"w":720,
"h":960,
"resize":"fit"
},
"thumb":{
"w":150,
"h":150,
"resize":"crop"
},
"small":{
"w":510,
"h":680,
"resize":"fit"
}
}
}
]
},
"favorited":false,
"retweeted":false,
"possibly_sensitive":false,
"filter_level":"low",
"lang":"en",
"timestamp_ms":"1485361639674",
"matching_rules":[
{
"tag":null,
"id":794211184496951296
}
]
}

1
project.clj
View file

@ -12,6 +12,7 @@
[cljs-ajax "0.8.0"]
[cljsjs/react-popper "0.10.4-0"]
[cljsjs/react-transition-group "2.4.0-0"]
[org.clojure/data.json "0.2.6"]
[clojure.java-time "0.3.2"]
[com.cognitect/transit-clj "0.8.313"]
[compojure "1.6.1"]

42
src/clj/ireadit/bot/responder.clj
View file

@ -11,14 +11,12 @@
[twitter.api.restful :as tar]
;; [twitter.api.streaming :as tas]
;; [twitter.callbacks :as tc]
;; [twitter.callbacks.handlers :as tch]
[twitter.callbacks.handlers :as tch]
[twitter.oauth :as to]
[twitter-streaming-client.core :as client]
;; )
;; (:import
;; ;; (twitter.callbacks.protocols SyncSingleCallback)
;; (twitter.callbacks.protocols SyncStreamingCallback)))
))
[twitter-streaming-client.core :as client])
(:import
(twitter.callbacks.protocols SyncSingleCallback)
(twitter.callbacks.protocols SyncStreamingCallback)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;
@ -82,9 +80,12 @@
(let [parent-id (:in_reply_to_status_id_str tweet)]
(if
parent-id
(tar/statuses-show :oauth-creds credentials
(tar/statuses-show-id :oauth-creds credentials
:params {:id parent-id
:include-entities true}))))
:include-entities true}
:callbacks (SyncSingleCallback. tch/response-return-body
tch/response-throw-error
tch/exception-rethrow)))))
(defn image-from-tweet
"Return the url of a media entity from this `tweet`, assumed to be a map
@ -94,7 +95,12 @@
([tweet]
(image-from-tweet tweet 0))
([tweet index]
(:media-url (get-in tweet (nth [:entities :media] index)))))
(:media_url
(nth
(or
(get-in tweet [:extended_entities :media])
(get-in tweet [:entities :media]))
index))))
(defn truncate
"If string `s` is longer than `n` characters, return a string like `s`
@ -112,7 +118,7 @@
[mention]
(let [screen-name (str "@" (get-in mention [:user :screen_name]))
image (image-from-tweet (tweet-replied-to [mention]))
trascription (if image
transcription (if image
(try
(str "Hi, "
screen-name
@ -129,12 +135,12 @@
"I'm sorry, I didn't find an image in that tweet.")]
(log/debug (str "replying to mention from " name))
(try
(statuses-update :oauth-creds credentials
(tar/statuses-update :oauth-creds credentials
:params {:status (truncate transcription 279)
:in_reply_to_status_id (:id_str mention)},
:callbacks (SyncSingleCallback. response-return-body
response-throw-error
exception-rethrow))
:in_reply_to_status_id (:id_str mention)}
:callbacks (SyncSingleCallback. tch/response-return-body
tch/response-throw-error
tch/exception-rethrow))
(catch Exception e (log/error e (str "error replying to mention from " (get-in mention [:user :screen_name])))))))
(defn reply-to-mentions
@ -151,10 +157,10 @@
(reply-to-mentions mentions)))
(defn bot []
(let [state (atom (assoc empty-state :minutes-since-update 30))
(let [;;state (atom (assoc empty-state :minutes-since-update 30))
previous (atom #{})]
(start-streams)
;;(start-streams)
(future (log/debug "STARTING USER STREAM")
(do-every 60500 handle-user-stream))))

2
src/cljs/ireadit/events.cljs
View file

@ -46,7 +46,7 @@
(rf/reg-event-db
:set-transcription
(fn [db [_ response]]
(js/console.log (str "Failed to fetch transcription data" response))
(js/console.log (str "Fetched transcription data `" response "`"))
(dissoc (assoc db :transcription response) :pending)))
(rf/reg-event-db

33
test/clj/ireadit/test/responder.clj Normal file
View file

@ -0,0 +1,33 @@
(ns ireadit.test.responder
(:require [clojure.test :refer :all]
[ring.mock.request :refer :all]
[ireadit.bot.responder :refer :all]
[muuntaja.core :as m]
[mount.core :as mount]))
(def tweet (read-string (slurp "env/test/resources/data/original_format_image_alt_text.edn")))
(image-from-tweet tweet 1)
(use-fixtures
:once
(fn [f]
(mount/start #'ireadit.config/env)
(f)))
(deftest test-image-url-extraction
(testing "Extraction of image URLs from Tweets"
(let [path (clojure.java.io/resource "/data/original_format_image_alt_text.edn")
p "env/test/resources/data/original_format_image_alt_text.edn"
tweet (read-string (slurp "env/test/resources/data/original_format_image_alt_text.edn"))]
(is (= p path))
(is (= tweet "froboz"))
(let [expected "http://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg"
actual (image-from-tweet tweet)]
(is (= expected actual)
(str "Failed to extract URL from tweet: expected `" expected "` found `" actual "`")))
(let [expected "http://pbs.twimg.com/media/C3B5jtZVIAAtCue.jpg"
actual (image-from-tweet tweet 1)]
(is (= expected actual)
(str "Failed to extract URL from tweet: expected `" expected "` found `" actual "`"))))))