angch.venice.1.12.16.source-code.tomcat-geoip-map.venice Maven / Gradle / Ivy
;;;; __ __ _
;;;; \ \ / /__ _ __ (_) ___ ___
;;;; \ \/ / _ \ '_ \| |/ __/ _ \
;;;; \ / __/ | | | | (_| __/
;;;; \/ \___|_| |_|_|\___\___|
;;;;
;;;;
;;;; Copyright 2017-2020 Venice
;;;;
;;;; Licensed under the Apache License, Version 2.0 (the "License");
;;;; you may not use this file except in compliance with the License.
;;;; You may obtain a copy of the License at
;;;;
;;;; http://www.apache.org/licenses/LICENSE-2.0
;;;;
;;;; Unless required by applicable law or agreed to in writing, software
;;;; distributed under the License is distributed on an "AS IS" BASIS,
;;;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;;;; See the License for the specific language governing permissions and
;;;; limitations under the License.
;;;;
;;;; Creates a HTML world map with the number of HTTP accesses per country.
;;;; The map can be viewed with a HTML browser.
;;;;
;;;; To lookup countries by IP addresses the MaxMind GeoLite databases is
;;;; required.
;;;; To download the free MaxMind GeoLite databases you need a license key that
;;;; can be obtained from the MaxMind 'https://www.maxmind.com/en/home' home
;;;; page.
;;;; Doc: https://dev.maxmind.com/geoip/geoip2/geoip2-city-country-csv-databases/
(do
(load-module :tomcat-util ['tomcat-util :as 'tc-util])
(load-module :geoip)
(load-module :kira)
;; The MaxMind country database
(def maxmind-country-zip "resources/geoip-country.zip")
;; A Trie Map with the private IPv4 Address Ranges (check private/public IPs)
(def private-ip4-trie (geoip/addr-ranges->trie geoip/private-ip4-addresses))
;; IP to country resolver
(def resolver (delay (create-resolver)))
;; SVG map template (full)
(def svg-map-tpl-full
"""
<%= (kira/escape-xml app-name) %> Access World Map <%= year %>
""")
;; SVG map template (light)
(def svg-map-tpl-light
"""
<%= (kira/escape-xml app-name) %> Access World Map <%= year %>
""")
(defn- print-help []
(println """
Actions:
[1] (lookup-ip "41.216.186.131")
[2] (download-maxmind-db "YOUR-MAXMIND-KEY")
[3] (run :full "STEP" 2022 "./ip-map.html"
"resources/logs/localhost_access_log.2022-12.zip")
[4] (run :full "STEP" 2022 "./ip-map.html"
"resources/logs/localhost_access_log.2022-12-01.log")
[5] (apply run :full "STEP" 2022 "./ip-map.html"
(io/list-files-glob "resources/logs/"
"localhost_access_log.2022-*"))
"""))
(defn- print-warn-macroexpand-on-load []
(println)
(print-msg-box :warn
"""
macroexpand-on-load is not activated. To get a better \
performance activate it before loading this script.
From the REPL run the command: !macroexpand
"""))
(defn- print-error-maxmind-data-missing []
(println)
(print-msg-box :error
"""
The MaxMind country file does not exist!
~{maxmind-country-zip}
Use `(download-maxmind-db "YOUR-MAXMIND-KEY")` to download it.
"""))
(defn create-resolver[]
(println "Loading Google country DB ...")
(let [country-db (geoip/download-google-country-db)]
(println "Parsing MaxMind DB ...")
(geoip/ip-to-country-loc-resolver maxmind-country-zip country-db)))
(defn public-ip? [ip]
(nil? (cidr/lookup-reverse private-ip4-trie ip)))
(defn app-req? [url]
(str/contains? url "/step/"))
(defn freq-map [ip status public-ip app-req]
{ :ip ip
:1xx (if (<= 100 status 199) 1 0)
:2xx (if (<= 200 status 299) 1 0)
:3xx (if (<= 300 status 399) 1 0)
:4xx (if (<= 400 status 499) 1 0)
:5xx (if (<= 500 status 599) 1 0)
:tot 1
:app (if app-req 1 0)
:public-ip public-ip
:country-iso nil })
(defn freq-aggregate [f1 f2]
{ :ip (:ip f2)
:1xx (+ (:1xx f1 0) (:1xx f2 0))
:2xx (+ (:2xx f1 0) (:2xx f2 0))
:3xx (+ (:3xx f1 0) (:3xx f2 0))
:4xx (+ (:4xx f1 0) (:4xx f2 0))
:5xx (+ (:5xx f1 0) (:5xx f2 0))
:tot (+ (:tot f1 0) (:tot f2 0))
:app (+ (:app f1 0) (:app f2 0))
:public-ip (:public-ip f2)
:country-iso (:country-iso f2) })
(defn frequencies-by-ip [items]
(reduce (fn [acc i] (->> (freq-aggregate (get acc (:ip i) {}) i)
(assoc acc (:ip i))))
{}
items))
(defn frequencies-by-country [items]
(reduce (fn [acc i] (->> (freq-aggregate (get acc (:country-iso i) {}) i)
(assoc acc (:country-iso i))))
{}
items))
(defn parse-ip-logs [log]
(let [parser (tc-util/default-access-log-entry-parser)]
(->> (bytebuf-to-string log :utf-8)
(str/split-lines)
(map parser)
(map #(freq-map (:ip %)
(or (:status %) 0)
(public-ip? (:ip %))
(app-req? (:url %))))
(frequencies-by-ip)
(vals))))
(defmulti parse-log-file (fn [f] (io/file-ext f)))
(defmethod parse-log-file "zip" [f]
(println "Parsing zip:" f "...")
(->> (io/unzip-all f)
(vals)
(map #(parse-ip-logs %))))
(defmethod parse-log-file "log" [f]
(println "Parsing log:" f "...")
(->> (io/slurp f :binary true)
(parse-ip-logs)))
(defn parse-log-files [log-files]
(->> (map parse-log-file log-files) ;; use pmap to parallelize
(flatten)))
(defn map-ip-to-country [item ip-country-resolver]
(let [location (if (:public-ip item) (ip-country-resolver (:ip item)) nil)]
(-> (dissoc item :ip :public-ip)
(assoc :country-iso (:country-iso location "PRIVATE")))))
(defn svg-country-data-full [item]
(let [country (:country-iso item)
s (str/format "total: %d, _1xx: %d, _2xx: %d, _3xx: %d, _4xx: %d, _5xx: %d, app: %d"
(:tot item) (:1xx item) (:2xx item)
(:3xx item) (:4xx item) (:5xx item)
(:app item))
prv (:private-ip item 0)]
(if (= country "CH")
(str/format "%s: { %s, _private: %d, color: '#2E9444' }" country s prv)
(str/format "%s: { %s, _private: 0 }" country s))))
(defn svg-country-data-light [item]
(let [country (:country-iso item)
tot (:tot item)
prv (:private-ip item 0)]
(if (= country "CH")
(str/format "%s: { total: %d, _private: %d, color: '#2E9444' }" country tot prv)
(str/format "%s: { total: %d, _private: 0 }" country tot))))
(defn svg-countries-data [mode app-name year items]
(let [max_ (->> (filter #(not (= (:country-iso %) "CH")) items)
(map :tot)
(apply max))
formatter (if (= mode :full) svg-country-data-full svg-country-data-light)]
{ :app-name app-name
:year year
:countries (str/join ", " (map formatter items))
:threshold-min 1
:threshold-max max_ }))
(defn create-html-svg-map [mode app-name year items]
(println "Creating HTML SVG map...")
(let [template (if (= mode :full) svg-map-tpl-full svg-map-tpl-light)
ch (first (filter #(= (:country-iso %) "CH") items))
private (first (filter #(= (:country-iso %) "PRIVATE") items))
others (filter #(nil? ((set "CH" "PRIVATE") (:country-iso %))) items)]
(cond
(and (nil? ch) (nil? private))
(->> others
(svg-countries-data mode app-name year)
(kira/eval template))
(and (some? ch) (some? private))
(->> (assoc (freq-aggregate private ch) :private-ip (:tot private 0))
(conj others)
(svg-countries-data mode app-name year)
(kira/eval template))
(some? ch)
(->> ch
(conj others)
(svg-countries-data mode app-name year)
(kira/eval template))
(some? private)
(->> (assoc private :country-iso "CH") ;; rename country to "CH"
(conj others)
(svg-countries-data mode app-name year)
(kira/eval template))
:else
(->> []
(svg-countries-data mode app-name year)
(kira/eval template)))))
(defn process [mode app-name year out-file log-files]
(println "Processing log files...")
(->> (map io/file log-files)
(parse-log-files) ;; parse log files -> entries
(frequencies-by-ip) ;; aggregate by IP address
(vals)
(map #(map-ip-to-country % @resolver)) ;; map IP to country
(frequencies-by-country) ;; aggregate by country
(vals)
(create-html-svg-map mode app-name year) ;; HTML with SVG map
(io/spit out-file)))
(defn download-maxmind-db [lic-key]
(->> (geoip/download-maxmind-db :country lic-key)
(io/spit (io/file maxmind-country-zip))))
(defn lookup-ip [ip] (@resolver ip))
(defn run [mode app-name year out-file & log-files]
(process mode app-name year out-file log-files))
;;; MAIN ---------------------------------------------------------------------
(print-help)
(when-not (macroexpand-on-load?)
(print-warn-macroexpand-on-load))
(when-not (io/exists-file? maxmind-country-zip)
(print-error-maxmind-data-missing)))