Analyse nginx logs

rsr-logs.clj

(ns workbench.rsr-logs
  (:require [cheshire.core :as json]
            [clojure.string :as s]
            [clojure-csv.core :as csv]
            [clj-time.coerce :as time-coerce]
            [clj-time.core :as time]
            [clj-time.format :as time-format]
            [clojure.contrib.humanize :as human]
            [clojure.string :as str]))


(def r
  (reverse
    (json/parse-string
      (slurp "/Users/dlebrero/Downloads/downloaded-logs-20210323-081822.json")
      true)))

(defn nice-time [t]
  (time-format/unparse
    (time-format/formatter "HH:mm:ss.SSS")
    t))

(->> r
  (filter (fn [x] (str/includes? (:textPayload x) " HTTP/1")))
  (remove (fn [x] (str/includes? (:textPayload x) " limiting request")))
  (remove (fn [x] (str/includes? (:textPayload x) " Connection refused")))
  (remove (fn [x] (str/includes? (:textPayload x) " Connection reset by peer")))
  (remove (fn [x] (str/includes? (:textPayload x) " Operation timed")))
  (remove (fn [x] (str/includes? (:textPayload x) "upstream prematurely")))
  (map
    (fn [{:keys [timestamp textPayload]}]
      (merge
        (-> textPayload
          (csv/parse-csv :delimiter \ )
          ((fn [x]
             (let [[ip _ _ t1 t2 url http-code response-size _ agent backend-time] (first x)]
               {:url (s/replace url #" *HTTP/1.*" "")
                :t1 t1
                :http-code http-code
                :response-size response-size
                :ip ip
                :agent agent
                :backend-time (Double/parseDouble backend-time)}))))
        {:end-time (time-coerce/from-string timestamp)})))
  (map
    (fn [{:keys [end-time backend-time] :as m}]
      (let [seconds (int backend-time)]
        (assoc m
          :start-time
          (time/minus end-time
            (time/seconds seconds)
            (time/millis (* 1000 (- backend-time seconds))))))))
  (filter (fn [{b :backend-time}] (> b 10)))
  ;(filter (fn [{b :url}] (str/starts-with? b "PATCH")))
  (map #(select-keys % [:url :http-code :backend-time :end-time :start-time :agent]))
  (map #(update % :end-time nice-time))
  (map #(update % :start-time nice-time))
  ;(map #(update % :url human/truncate 50))
  (sort-by :start-time)
  (clojure.pprint/print-table))