Compare serialization size between transit, nippy and edn

measure_memory.clj

(ns local.measure-memory
  {:clj-kondo/config '{:linters :ignore}}
  (:require
   [clj-memory-meter.core :as mm]
   [clojure.edn :as edn]
   [clojure.java.io :as io]
   [cognitect.transit :as transit]
   [metabase.query-processor :as qp]
   [metabase.util.json :as json]
   [taoensso.nippy :as nippy]
   [toucan2.core :as t2])
  (:import
   (java.io File FileWriter FileReader)))

;; ANALYTICS EVENTS table with 36 k rows
(def table (t2/select-one :model/Table :name "ANALYTIC_EVENTS"))
(def rows (-> (qp/process-query {:database (:db_id table)
                                 :type     :query
                                 :query    {:source-table (:id table)}})
              :data
              :rows))

(first rows)
;; => [1 83 "Button Clicked" "2022-03-15T00:18:25Z" nil "Invite" "[email protected]"]

(count rows)
;; => 37169

(mm/measure rows)
;; => "7.0 MiB"

;; -------------------------- TRANSIT --------------------------

(defn to-transit [data]
  (let [file (java.io.File/createTempFile "transit-" ".json")]
    (with-open [out (java.io.FileOutputStream. file)]
      (let [writer (transit/writer out :json)]
        (transit/write writer data)))
    file)) ;; Return the file object

(defn read-transit [file]
  (with-open [in (java.io.FileInputStream. file)]
    (let [reader (transit/reader in :json)]
      (transit/read reader)))) ;; Return the data

(-> rows to-transit read-transit mm/measure time)
;; => "20.3 MiB"
;; "Elapsed time: 366.172625 msecs"
(= rows (-> rows to-transit read-transit))
;; => true

;; -------------------------- NIPPY --------------------------
(-> rows nippy/freeze nippy/thaw mm/measure time)
;; => "13.8 MiB"
;; "Elapsed time: 83.165 msecs"
(= rows (-> rows nippy/freeze nippy/thaw))

;; -------------------------- JSON String --------------------------

(defn- to-json
  [rows]
  (spit "data.json" (json/encode rows))
  "data.json")

(defn- read-json
  [fname]
  (json/decode (slurp fname)))

;; => "6.5 MiB"
(-> rows to-json read-json mm/measure time)
;; => "3.9 MiB"
;; "Elapsed time: 47.698084 msecs"
(= rows (-> rows to-json read-json))
;; => true

;; -------------------------- EDN --------------------------

(defn to-edn [data]
  (let [file (File/createTempFile "data-" ".edn")]
    (with-open [w (FileWriter. file)]
      (.write w (pr-str data))) ;; Write EDN to file
    file)) ;; Return the file object

(defn read-edn [file]
  (with-open [r (io/reader file)]
    (edn/read (java.io.PushbackReader. r)))) ;; Read EDN from file

(-> rows to-edn read-edn mm/measure time)
;; => "11.0 MiB"
;; "Elapsed time: 166.334541 msecs"

(= rows (-> rows to-edn read-edn))
;; => true