angch.venice.1.12.34.source-code.covid-19.venice Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of venice Show documentation
Show all versions of venice Show documentation
Venice, a sandboxed Lisp implemented in Java.
The newest version!
;;;; __ __ _
;;;; \ \ / /__ _ __ (_) ___ ___
;;;; \ \/ / _ \ '_ \| |/ __/ _ \
;;;; \ / __/ | | | | (_| __/
;;;; \/ \___|_| |_|_|\___\___|
;;;;
;;;;
;;;; Copyright 2017-2024 Venice
;;;;
;;;; Licensed under the Apache License, Version 2.0 (the "License");
;;;; you may not use this file except in compliance with the License.
;;;; You may obtain a copy of the License at
;;;;
;;;; http://www.apache.org/licenses/LICENSE-2.0
;;;;
;;;; Unless required by applicable law or agreed to in writing, software
;;;; distributed under the License is distributed on an "AS IS" BASIS,
;;;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;;;; See the License for the specific language governing permissions and
;;;; limitations under the License.
;;;; Creates a Covid-19 chart representing Switzerland compared to some other
;;;; countries
;;;;
;;;; Composes the charts from three data sets that are available on the
;;;; internet:
;;;; 1. Covid-19 country cases https://www.worldometers.info/coronavirus/
;;;; 2. Covid-19 country vaccinations https://github.com/owid/covid-19-data/
;;;; 3. Country population https://restcountries.eu/ (offline)
;;;; https://restcountries.com/
;;;; https://github.com/apilayer/restcountries/
(do
(ns covid-19)
(load-module :ansi)
(load-module :xchart)
(def country-set-1 [
"Switzerland" "Romania" "Austria" "Hungary"
"Germany" "Italy" "France" "Sweden"
"Greece" "Spain" "UK" "Netherlands"
"Iceland" "Denmark" ])
(def country-set-2 [
"Gibraltar" "Croatia" "Portugal" "Belgium"
"Luxembourg" "Norway" "Finland" "Israel"
"Ireland" "Slovenia" "Poland" "US"
"Czech-Republic" "Turkey" ])
(def country-set-3 [
"India" "Australia" "China" "Russia"
"Lithuania" "Japan" "Thailand" "Bulgaria"
"Singapore" "New-Zealand" "Brazil" "Argentina"
"Colombia" "Canada" ])
(def country-set-4 [
"Ukraine" "Slovakia" "Tunisia" ])
;; 14 styles for the chart series, each visualizing a country
(def chart-series-styles [
{ :color :x-green }
{ :color :x-blue }
{ :color :x-purple }
{ :color :x-orange }
{ :color :x-red }
{ :color :x-pink }
{ :color :x-light-blue }
{ :color :x-dark-gray }
{ :color :x-dark-pink }
{ :color :x-dark-green }
{ :color :x-yellow }
{ :color :x-magenta }
{ :color :x-black }
{ :color :x-brown } ])
;; country - iso code mappings
(def country-iso-mapping [
{ :name "Argentina" :iso "ARG" }
{ :name "Australia" :iso "AUS" }
{ :name "Austria" :iso "AUT" }
{ :name "Belgium" :iso "BEL" }
{ :name "Brazil" :iso "BRA" }
{ :name "Bulgaria" :iso "BGR" }
{ :name "Canada" :iso "CAN" }
{ :name "China" :iso "CHN" }
{ :name "Colombia" :iso "COL" }
{ :name "Croatia" :iso "HRV" }
{ :name "Czech-Republic" :iso "CZE" }
{ :name "Denmark" :iso "DNK" }
{ :name "Estonia" :iso "EST" }
{ :name "Finland" :iso "FIN" }
{ :name "France" :iso "FRA" }
{ :name "Germany" :iso "DEU" }
{ :name "Gibraltar" :iso "GIB" }
{ :name "Greece" :iso "GRC" }
{ :name "Hungary" :iso "HUN" }
{ :name "Iceland" :iso "ISL" }
{ :name "India" :iso "IND" }
{ :name "Ireland" :iso "IRL" }
{ :name "Israel" :iso "ISR" }
{ :name "Italy" :iso "ITA" }
{ :name "Japan" :iso "JPN" }
{ :name "Latvia" :iso "LVA" }
{ :name "Lithuania" :iso "LTU" }
{ :name "Luxembourg" :iso "LUX" }
{ :name "Netherlands" :iso "NLD" }
{ :name "New-Zealand" :iso "NZL" }
{ :name "Norway" :iso "NOR" }
{ :name "Poland" :iso "POL" }
{ :name "Portugal" :iso "PRT" }
{ :name "Romania" :iso "ROU" }
{ :name "Russia" :iso "RUS" }
{ :name "Singapore" :iso "SGP" }
{ :name "Slovenia" :iso "SVN" }
{ :name "Slovakia" :iso "SVK" }
{ :name "Spain" :iso "ESP" }
{ :name "South Africa" :iso "ZAF" }
{ :name "Sweden" :iso "SWE" }
{ :name "Switzerland" :iso "CHE" }
{ :name "Thailand" :iso "THA" }
{ :name "Tunisia" :iso "TUN" }
{ :name "Turkey" :iso "TUR" }
{ :name "UK" :iso "GBR" }
{ :name "Ukraine" :iso "UKR" }
{ :name "US" :iso "USA" } ])
;; the interval length in days to calculate the moving average
(def sample-days 7)
;; the start/end date for the cases
(def start-date (time/local-date 2021 10 1))
(def end-date nil)
;; chart rendering
(def chart-width 700)
(def chart-height 500)
(def chart-dpi 240)
;; origin of the corona case data
(def worldometers-url "https://www.worldometers.info/coronavirus/country/")
;; country population REST URL
; (def rest-population-url "https://restcountries.eu/rest/v2/alpha/")
(def rest-population-url "https://restcountries.com/v2/alpha/")
;; Covid-19 vaccination data
;; https://www.google.com/search?q=schweiz+covid
;; https://www.google.com/search?q=schweiz+covid+impfung
;; https://ourworldindata.org/covid-vaccinations
;; https://github.com/owid/covid-19-data/tree/master/public/data/vaccinations
(def vaccinations-url "https://raw.githubusercontent.com/owid/covid-19-data/master/public/data/vaccinations/vaccinations.json")
;;; I/O functions (not pure) -------------------------------------------------
(defn view-chart! [file] (sh/open file))
(defn save-chart! [chart file dpi]
(println (ansi/style "Saving chart ~{file}" (ansi/fg-color 235)))
(xchart/write-to-file chart :png dpi file)
file)
(defn load-country-html-data! [country-name]
(let [caption (str/expand "Loading case data for ~{country-name}:" 38 " ")
url (str worldometers-url (str/lower-case country-name) "/")]
(ansi/without-cursor
(io/download url
:user-agent "Mozilla"
:progress-fn (ansi/progress :caption caption
:end-col :green
:failed-col :red)))))
(defn load-countries-html-data! [country-names]
(->> (map load-country-html-data! country-names)
(zipmap country-names)))
(defn load-country-population! [country-iso]
(let [url (str rest-population-url country-iso)]
(-<> (io/download url :user-agent "Mozilla")
(json/read-str <>)
(get <> "population")
(long <>))))
(defn load-countries-population! [countries-iso]
(let [caption (str/expand "Loading country population data:" 38 " ")
counter (atom 0.0)
delta (/ 100.0 (count countries-iso))
pg (ansi/progress :caption caption
:end-col :green
:failed-col :red)]
(ansi/without-cursor
(try
(pg 0 :progress)
(->> (map (fn [x]
(let [population (load-country-population! x)]
(pg (swap! counter #(+ % delta)) :progress)
population))
countries-iso)
(zipmap countries-iso))
(finally
(pg 100 :progress)
(pg 100 :end))))))
(def load-country-vaccinations!
;; returns a map with country iso code and vaccinated (all doses) rate
;; { "CHE" 54.18, "AUT" 58.97, ... }
;;
;; JSON
;; {
;; "country": "Switzerland",
;; "iso_code": "CHE",
;; "data": [
;; {
;; "date": "2021-12-01",
;; "total_vaccinations": 11983616,
;; "people_vaccinated": 5851536,
;; "people_fully_vaccinated": 5708838,
;; "total_boosters": 590055,
;; "daily_vaccinations_raw": 66275,
;; "daily_vaccinations": 43832,
;; "total_vaccinations_per_hundred": 137.5,
;; "people_vaccinated_per_hundred": 67.14,
;; "people_fully_vaccinated_per_hundred": 65.5,
;; "total_boosters_per_hundred": 6.77,
;; "daily_vaccinations_per_million": 5029,
;; "daily_people_vaccinated": 3223,
;; "daily_people_vaccinated_per_hundred": 0.037
;; }
(memoize
(fn []
(let [caption (str/expand "Loading country vaccination data:" 38 " ")
format #(str/format "%.1f%%" %)]
(ansi/without-cursor
(->> (io/download vaccinations-url
:user-agent "Mozilla"
:progress-fn (ansi/progress :caption caption
:end-col :green
:failed-col :red))
(json/read-str)
(map #(list (get % "iso_code")
(format (get (last (get % "data"))
"people_fully_vaccinated_per_hundred"
0.00))))
(flatten)
(apply hash-map)))))))
;;; Extracting and parse cases from HTML data --------------------------------
(defn extract-html-daily-cases-section [html]
;; extract the daily cases section from HTML page (javascript)
(->> (str/split-lines html)
(drop-while #(not (str/contains? % "'graph-cases-daily'" )))
(take-while #(not (str/contains? % "});" )))
(rest)
(butlast)))
(defn parse-daily-cases-dates [html]
;; """categories: ["Feb 15, 2020","Feb 16, 2020",...,"Aug 29, 2020"] },"""
(as-> html <>
(filter #(str/contains? % "categories: [") <>)
(first <>)
(regex/find-all (regex/matcher """["][A-Za-z0-9, ]+["]""" <>))
(map str/double-unquote <>)
(map #(time/local-date-parse % "MMM dd, yyyy" :ENGLISH) <>)))
(defn parse-daily-cases-numbers [html]
;; """data: [null,0,0,0,0,0,1,3,4,...,215,186,143,121] },"""
(as-> html <>
(filter #(str/contains? % "data: [") <>)
(first <>)
(str/replace-all <> "null" "0")
(regex/find-all (regex/matcher "\\d+" <>))
(map long <>)))
(defn parse-country-cases [html]
;; Returns a list with tuples of date and cases
;; (... [2020-05-21 36] [2020-05-22 13] [2020-05-23 18] ... )
;;
;; (->> (load-country-html-data! "Switzerland")
;; (parse-country-cases))
(let [section (extract-html-daily-cases-section html)]
(map vector (parse-daily-cases-dates section)
(parse-daily-cases-numbers section))))
;;; Aggregate cases ----------------------------------------------------------
(defn add-country-cases [country-data]
(-> country-data
(assoc :cases (parse-country-cases (:html country-data)))
(dissoc :html)))
(defn aggregate-cases [normalized-population cases]
(->> (partition sample-days 1 cases) ;; create windows of sample-days len
(map (fn [window]
(let [date (first (last window))
tot (reduce + (map second window))
avg (/ tot normalized-population)]
[date (long avg)])))))
(defn aggregate-country-cases [country-data]
(->> (:cases country-data)
(aggregate-cases (:population-normalized country-data))
(assoc country-data :cases)))
(defn filter-country-cases-by-date [start end country-data]
(let [in-period? #(time/within? (first %) start end)]
(assoc country-data :cases (filter in-period? (:cases country-data)))))
;;; Aggregate country data ---------------------------------------------------
(defn map-country-iso-mapping-by-name []
(reduce #(assoc %1 (:name %2) %2) {} country-iso-mapping))
(defn country-names-to-iso [country-names]
(let [country-map (map-country-iso-mapping-by-name)]
(map #(:iso (get country-map %)) country-names)))
(defn enrich-country-data-with-chart-styles [country-data]
(let [styles (map #(hash-map :style %) chart-series-styles)]
(map merge country-data styles)))
(defn enrich-country-data-with-vaccination-rate [country-vacs country-data]
(let [iso-list (map :iso country-data)]
(map #(assoc % :vac (get country-vacs (:iso %))) country-data)))
(defn enrich-country-data-with-population [populations country-data]
(let [iso-list (map :iso country-data)]
(->> country-data
(map #(assoc % :population
(get populations (:iso %))))
(map #(assoc % :population-normalized
(/ (get populations (:iso %)) 100_000.0))))))
(defn enrich-country-data-with-cases-html [countries-html country-data]
(map #(assoc % :html (get countries-html (:name %))) country-data))
(defn build-country-data [country-names]
;; build a list of maps of { :name "Switzerland" :iso "CHE" }
(let [country-map (map-country-iso-mapping-by-name)]
(map #(get country-map %) country-names)))
(defn aggregate-country-data [country-names start end
country-vacs populations countries-html]
(->> (build-country-data country-names)
(enrich-country-data-with-chart-styles)
(enrich-country-data-with-vaccination-rate country-vacs)
(enrich-country-data-with-population populations)
(enrich-country-data-with-cases-html countries-html)
(map #(add-country-cases %))
(map #(aggregate-country-cases %))
(map #(filter-country-cases-by-date start end %))))
;;; Chart --------------------------------------------------------------------
(defn chart-series-country [country-data]
(let [cases (:cases country-data)
style (:style country-data)]
{ :x (map (comp time/date first) cases)
:y (map second cases)
:style { :marker-type :none
:line-color (:color style)
:line-style :solid }}))
(defn chart-series-all-countries [countries-data]
(let [series-name (fn [data] (str (:name data) " - " (:vac data)))]
(reduce #(assoc %1 (series-name %2) (chart-series-country %2))
(ordered-map)
countries-data)))
(defn render-chart [countries-data width height sample-days]
(xchart/xy-chart
(chart-series-all-countries countries-data)
{ :title "Covid-19 Incidence (~(str (time/local-date)))"
:width width
:height height
:x-axis { :title "", :date-pattern "MMM-dd" }
:y-axis { :title "Cases over ~{sample-days} Days per 100'000" }
:theme :matlab
:legend { :border-color :white }
:render-style :line } ))
;;; Process ------------------------------------------------------------------
(defn process! [filename country-names]
;; 1) Download data from Web (not pure)
;; 2) Aggregate the data (pure)
;; 3) Create the chart (pure)
;; 4) Save and open the chart (not pure)
(let [countries-iso (country-names-to-iso country-names)
vaccinations (load-country-vaccinations!)
populations (load-countries-population! countries-iso)
countries-html (load-countries-html-data! country-names)]
(-<> (aggregate-country-data country-names
start-date
end-date
vaccinations
populations
countries-html)
(render-chart <> chart-width chart-height sample-days)
(save-chart! <> filename chart-dpi)
(view-chart! <>))))
;;; Main ---------------------------------------------------------------------
(process! "covid-19_1.png" country-set-1)
(process! "covid-19_2.png" country-set-2)
(process! "covid-19_3.png" country-set-3)
;(process! "covid-19_4.png" country-set-4)
)