Compare commits
2 commits
73549a5c90
...
86665db3b8
| Author | SHA1 | Date | |
|---|---|---|---|
| 86665db3b8 | |||
| ea2d6927d2 |
5 changed files with 43 additions and 5 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
|
@ -13,4 +13,6 @@ pom.xml.asc
|
|||
.lein-failures
|
||||
.nrepl-port
|
||||
.cpcache/
|
||||
|
||||
.lsp/
|
||||
.clj-kondo/
|
||||
.portal/
|
||||
|
|
@ -3,7 +3,8 @@
|
|||
:url "http://example.com/FIXME"
|
||||
:license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
|
||||
:url "https://www.eclipse.org/legal/epl-2.0/"}
|
||||
:dependencies [[org.clojure/clojure "1.11.1"]
|
||||
:dependencies [[dev.weavejester/medley "1.9.0"]
|
||||
[org.clojure/clojure "1.11.1"]
|
||||
[peco "0.1.6"]]
|
||||
:repl-options {:init-ns cc.journeyman.elboob.core}
|
||||
:source-paths ["src/clj"])
|
||||
|
|
|
|||
1
resources/ignorable-words.en-GB.edn
Symbolic link
1
resources/ignorable-words.en-GB.edn
Symbolic link
|
|
@ -0,0 +1 @@
|
|||
ignorable-words.en.edn
|
||||
|
|
@ -1,9 +1,43 @@
|
|||
(ns cc.journeyman.elboob.core)
|
||||
(ns cc.journeyman.elboob.core
|
||||
(:require [clojure.java.io :refer [as-relative-path file resource]]
|
||||
[clojure.string :refer [ends-with?]]
|
||||
[medley.core :refer [deep-merge]]
|
||||
[peco.core :refer [tokenizer]]))
|
||||
|
||||
(defn compile
|
||||
(defn get-ignorable-words []
|
||||
(let [locale (java.util.Locale/getDefault)
|
||||
tag (.toLanguageTag locale)
|
||||
language (.getLanguage locale)]
|
||||
|
||||
(first
|
||||
(map #(try (println (format "resources/ignorable-words%s.edn" %))
|
||||
(read-string
|
||||
(slurp
|
||||
(file (format "resources/ignorable-words%s.edn" %))))
|
||||
(catch Exception e (println (.getMessage e)) nil))
|
||||
[(str "." tag) (str "." language) ""]))))
|
||||
|
||||
(defn compile-file
|
||||
"Compile an index for an individual file `f`, tokenised with `tokenise` and
|
||||
filtered with `ignorable?`."
|
||||
[f tokenise ignorable?]
|
||||
(let [f' (file f)
|
||||
rel (as-relative-path f')
|
||||
tokens (frequencies (remove ignorable? (tokenise (slurp f'))))]
|
||||
(reduce #(assoc %1 %2 {rel (tokens %2)}) {} (keys tokens))))
|
||||
|
||||
(defn compile-index
|
||||
"scans `dir-paths` as directories of Markdown files. Returns a map which keys
|
||||
each lexical token occurring in each file (with Markdown formatting, common
|
||||
words, punctuation etc excepted) to a map which keys the relative file path
|
||||
of each file in which the token occurs to the frequency the token occurs within the file."
|
||||
[& dir-paths]
|
||||
(println "Hello, World!"))
|
||||
(let [ignorable-word? (set (get-ignorable-words))
|
||||
tokenise (tokenizer [:lower-case :concat-singles])]
|
||||
(reduce deep-merge {}
|
||||
(map (fn [dir]
|
||||
(map #(compile-file % tokenise ignorable-word?)
|
||||
(filter
|
||||
#(ends-with? (.getName %) ".md")
|
||||
(file-seq (file dir))))) dir-paths))))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue