janet bench/test for dirge janet compression plugin

bench.janet

# A/B compression benchmark — measures token reduction across all
# supported command families with realistic shell output.
#
# Usage: janet .dirge/plugins/compression/bench.janet

# ── Bootstrap (same as test.janet) ──────────────────────────────────────────
(var harness-replace-result nil)
(defn harness/replace-result [result] (set harness-replace-result result))
(defn loadf [path] (eval-string (slurp path)))

(loadf ".dirge/plugins/compression/00-regex.janet")
(loadf ".dirge/plugins/compression/10-git.janet")
(loadf ".dirge/plugins/compression/20-cargo.janet")
(loadf ".dirge/plugins/compression/30-docker.janet")
(loadf ".dirge/plugins/compression/40-kubectl.janet")
(loadf ".dirge/plugins/compression/45-npm.janet")
(loadf ".dirge/plugins/compression/50-misc.janet")
(loadf ".dirge/plugins/compression/init.janet")

# ── Realistic output samples ────────────────────────────────────────────────

(def SAMPLES
  @[
    # ── git ──
    @{:family "git" :label "status (clean)"
      :command "git status"
      :output "On branch main\nYour branch is up to date with 'origin/main'.\n\nnothing to commit, working tree clean"}

    @{:family "git" :label "status (dirty)"
      :command "git status"
      :output "On branch feature/foo\nYour branch is ahead of 'origin/main' by 3 commits.\n  (use \"git push\" to publish your local commits)\n\nChanges to be committed:\n  (use \"git restore --staged <file>...\" to unstage)\n\tmodified:   src/main.rs\n\tnew file:   src/plugin.janet\n\nChanges not staged for commit:\n  (use \"git add <file>...\" to update what will be committed)\n  (use \"git restore <file>...\" to discard changes in working directory)\n\tmodified:   README.md\n\nUntracked files:\n  (use \"git add <file>...\" to include in what will be committed)\n\tTODO.txt"}

    @{:family "git" :label "diff --stat"
      :command "git diff --stat"
      :output " src/main.rs  | 42 ++++++++++++++++++++++++++--------\n src/lib.rs   | 15 +++++++------\n 2 files changed, 35 insertions(+), 8 deletions(-)"}

    # ── cargo ──
    @{:family "cargo" :label "build (success)"
      :command "cargo build"
      :output "   Compiling dirge v0.1.0 (/home/user/projects/dirge)\n    Finished dev [unoptimized + debuginfo] target(s) in 12.34s"}

    @{:family "cargo" :label "test"
      :command "cargo test"
      :output "running 42 tests\ntest agent::tools::read::tests::test_line_number_format ... ok\ntest agent::tools::cache::tests::test_cache_hit_and_miss ... ok\ntest agent::tools::cache::tests::test_cache_clear_invalidates_entries ... ok\n... (omitting 36 passing tests)\ntest extras::session_db::tests::create_and_read_session ... ok\n\ntest result: ok. 42 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.15s"}

    @{:family "cargo" :label "clippy (warnings)"
      :command "cargo clippy"
      :output "    Checking dirge v0.1.0\nwarning: unnecessary closure\n  --> src/main.rs:42:10\n   |\n42 |     let x = (0..10).map(|n| n + 1).collect::<Vec<_>>();\n   |                     ^^^^^^^^^^^^^^ help: use the function directly: `i32::wrapping_add`\n   |\n   = note: `#[warn(clippy::redundant_closure)]` on by default\n\nwarning: `dirge` (bin \"dirge\") generated 1 warning\n    Finished dev [unoptimized + debuginfo] target(s) in 3.21s"}

    # ── docker ──
    @{:family "docker" :label "ps"
      :command "docker ps"
      :output "CONTAINER ID   IMAGE          COMMAND                  CREATED        STATUS        PORTS                    NAMES\nabc123def456   nginx:latest   \"/docker-entrypoint.…\"   2 hours ago    Up 2 hours    0.0.0.0:8080->80/tcp     webserver\ndef789abc012   redis:alpine   \"docker-entrypoint.s…\"   5 days ago     Up 5 days     6379/tcp                 cache\n111222333444   postgres:15    \"docker-entrypoint.s…\"   1 week ago     Up 1 week     0.0.0.0:5432->5432/tcp   db"}

    @{:family "docker" :label "images"
      :command "docker images"
      :output "REPOSITORY      TAG        IMAGE ID       CREATED        SIZE\nnginx           latest     abc123def456   2 weeks ago    187MB\nredis           alpine     def789abc012   3 weeks ago    32.4MB\npostgres        15         111222333444   4 weeks ago    425MB\nnode            20-slim    555666777888   5 weeks ago    248MB"}

    # ── kubectl ──
    @{:family "kubectl" :label "get pods"
      :command "kubectl get pods"
      :output "NAME                          READY   STATUS    RESTARTS   AGE\napi-deployment-6d4b7c8f9-abc   1/1     Running   0          2d\napi-deployment-6d4b7c8f9-def   1/1     Running   0          2d\nworker-deployment-7e5f8a9b-ghi 1/1     Running   1          5d\nredis-8f6a9b0c-jkl             1/1     Running   0          7d"}

    @{:family "kubectl" :label "describe pod"
      :command "kubectl describe pod api-deployment-6d4b7c8f9-abc"
      :output "Name:             api-deployment-6d4b7c8f9-abc\nNamespace:        default\nPriority:         0\nNode:             minikube/192.168.49.2\nStart Time:       Mon, 01 Jan 2026 10:00:00 +0000\nLabels:           app=api\n                  pod-template-hash=6d4b7c8f9\nAnnotations:      kubectl.kubernetes.io/restartedAt: 2026-01-01T10:00:00Z\nStatus:           Running\nIP:               10.244.0.5\nIPs:\n  IP:           10.244.0.5\nControlled By:  ReplicaSet/api-deployment-6d4b7c8f9\nContainers:\n  api:\n    Container ID:   docker://abc123def456\n    Image:          api:latest\n    Port:           8080/TCP\n    State:          Running\n      Started:      Mon, 01 Jan 2026 10:00:00 +0000\n    Ready:          True\nConditions:\n  Type              Status\n  Initialized       True\n  Ready             True\n  ContainersReady   True\n  PodScheduled      True"}

    # ── npm ──
    @{:family "npm" :label "install (many deps)"
      :command "npm install"
      :output "npm WARN deprecated package@1.0.0: Use package2 instead\n\nadded 247 packages from 128 contributors in 45.3s\n\n45 packages are looking for funding\n  run `npm fund` for details"}

    @{:family "npm" :label "outdated"
      :command "npm outdated"
      :output "Package     Current  Wanted  Latest  Location\nexpress     4.18.0   4.18.2  5.0.0   dirge\nlodash      4.17.15  4.17.21 4.17.21 dirge\ntypescript  5.0.0    5.0.4   5.3.2   dirge"}

    # ── pip ──
    @{:family "pip" :label "install packages"
      :command "pip install requests"
      :output "Collecting requests\n  Downloading requests-2.31.0-py3-none-any.whl (62 kB)\n     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.3/62.3 kB 1.2 MB/s eta 0:00:00\nCollecting certifi>=2017.4.17\n  Downloading certifi-2024.2.2-py3-none-any.whl (150 kB)\n     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 150.5/150.5 kB 2.1 MB/s eta 0:00:00\nCollecting charset-normalizer<4,>=3\n  Downloading charset_normalizer-3.3.2-py3-none-any.whl (48 kB)\n     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 48.2/48.2 kB 3.4 MB/s eta 0:00:00\nCollecting idna<4,>=2.5\n  Downloading idna-3.4-py3-none-any.whl (61 kB)\n     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.1/61.1 kB 2.8 MB/s eta 0:00:00\nCollecting urllib3<3,>=1.21.1\n  Downloading urllib3-2.0.5-py3-none-any.whl (123 kB)\n     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 123.4/123.4 kB 3.1 MB/s eta 0:00:00\nInstalling collected packages: urllib3, idna, charset-normalizer, certifi, requests\nSuccessfully installed certifi-2024.2.2 charset-normalizer-3.3.2 idna-3.4 requests-2.31.0 urllib3-2.0.5"}

    # ── grep ──
    @{:family "grep" :label "many matches"
      :command "grep -r TODO src/"
      :output (string/join
                (map (fn [i] (string "src/file" i ".rs:123: TODO: implement this"))
                     (range 1 41))
                "\n")}

    # ── find ──
    @{:family "find" :label "many results"
      :command "find . -name '*.rs'"
      :output (string/join
                (map (fn [i] (string "./src/module" i "/file.rs"))
                     (range 1 31))
                "\n")}

    # ── ls ──
    @{:family "ls" :label "large directory"
      :command "ls -la"
      :output (let [entries @[]]
                (array/push entries "total 1024")
                (for i 1 51
                  (array/push entries (string "-rw-r--r-- 1 user user 1024 Jan " i " 12:00 file" i ".txt")))
                (string/join entries "\n"))}

    # ── curl ──
    @{:family "curl" :label "JSON API response"
      :command "curl https://api.example.com/users"
      :output "[\n  {\"id\": 1, \"name\": \"Alice\", \"email\": \"alice@example.com\"},\n  {\"id\": 2, \"name\": \"Bob\", \"email\": \"bob@example.com\"},\n  {\"id\": 3, \"name\": \"Charlie\", \"email\": \"charlie@example.com\"}\n]"}
  ])

# ── Routing table (family → compressor) ─────────────────────────────────────

(def COMPRESSORS
  {:git git-compress
   :cargo cargo-compress
   :docker docker-compress
   :kubectl kubectl-compress
   :npm npm-compress
   :pip pip-compress
   :grep grep-compress
   :find find-compress
   :ls ls-compress
   :curl curl-compress})

# ── Thresholds ───────────────────────────────────────────────────────────────
# Minimum expected reduction per family. Stat-only/summary outputs
# (e.g. git diff --stat, small curl responses) legitimately can't
# be compressed, so thresholds are set to the worst realistic case.

(def THRESHOLDS
  {:git {:min-reduction 0.0}     # diff --stat is already terse
   :cargo {:min-reduction 0.50}
   :docker {:min-reduction 0.30}
   :kubectl {:min-reduction 0.30}
   :npm {:min-reduction 0.15}    # outdated with few pkgs has low reduction
   :pip {:min-reduction 0.70}
   :grep {:min-reduction 0.40}
   :find {:min-reduction 0.40}
   :ls {:min-reduction 0.50}
   :curl {:min-reduction 0.0}})  # small responses pass through

# ── Benchmark runner ─────────────────────────────────────────────────────────

(print "=")
(print "Compression Benchmark")
(print "=")
(print)

(var total-orig 0)
(var total-comp 0)
(var total-samples 0)
(var failures @[])

(each sample SAMPLES
  (let [family (sample :family)
        label (sample :label)
        command (sample :command)
        output (sample :output)
        compressor (COMPRESSORS (keyword family))
        orig-len (length output)
        compressed (compressor command output)
        comp-len (if (nil? compressed) orig-len (length compressed))]

    (when (nil? compressed)
      (array/push failures (string family " " label ": compressor returned nil (passthrough)")))

    (def reduction (if (> orig-len 0) (- 1 (/ comp-len orig-len)) 0))
    (def pct (math/round (* reduction 100)))
    (def threshold (get-in THRESHOLDS [(keyword family) :min-reduction] 0))
    (def status (if (>= reduction threshold) "✓" "✗"))

    (when (< reduction threshold)
      (array/push failures
        (string family " " label ": " (math/round (* reduction 100))
                "% < " (math/round (* threshold 100)) "% threshold")))

    (print (string status " " family "/" label))
    (print (string "  " orig-len " → " comp-len " chars (" pct "% reduction, threshold " (math/round (* threshold 100)) "%)"))

    (+= total-orig orig-len)
    (+= total-comp comp-len)
    (++ total-samples)))

(print)
(print "---")
(def overall-reduction (if (> total-orig 0) (- 1 (/ total-comp total-orig)) 0))
(print (string "Total: " total-orig " → " total-comp " chars across " total-samples " samples"))
(print (string "Overall reduction: " (math/round (* overall-reduction 100)) "%"))

(when (not (empty? failures))
  (print)
  (print "FAILURES:")
  (each f failures (print "  " f)))

(print)
(if (empty? failures)
  (print "All thresholds met.")
  (do
    (print (string (length failures) " threshold(s) not met."))
    (os/exit 1)))

test.janet

# Test script for compression plugins
# Loads all files in alphabetical order (as dirge does)

# Stub harness/replace-result
(var harness-replace-result nil)
(defn harness/replace-result [result]
  (set harness-replace-result result))

# Load plugin files in order
(defn loadf [path]
  (eval-string (slurp path)))

(loadf ".dirge/plugins/compression/00-regex.janet")
(loadf ".dirge/plugins/compression/10-git.janet")
(loadf ".dirge/plugins/compression/20-cargo.janet")
(loadf ".dirge/plugins/compression/30-docker.janet")
(loadf ".dirge/plugins/compression/40-kubectl.janet")
(loadf ".dirge/plugins/compression/45-npm.janet")
(loadf ".dirge/plugins/compression/50-misc.janet")
(loadf ".dirge/plugins/compression/init.janet")

(defn assert-str [label expected got]
  (if (= expected got)
    (print "✓ " label)
    (do
      (print "✗ " label)
      (print "  expected: " (if (nil? expected) "nil" (string/replace-all "\n" "⏎" expected)))
      (print "  got:      " (if (nil? got) "nil" (string/replace-all "\n" "⏎" got))))))

(defn test-compressor [label compressor command input expected]
  (assert-str label expected (compressor command input)))

# ── pip ──
(test-compressor
  "pip install success"
  pip-compress "pip install requests"
  "Collecting requests\n  Downloading requests-2.28.0.tar.gz\nSuccessfully installed requests certifi charset-normalizer idna urllib3"
  "installed 5 packages: requests, certifi, charset-normalizer, idna, urllib3")

(test-compressor
  "pip non-pip command"
  pip-compress "git status"
  "nothing to commit"
  nil)

# ── grep ──
(test-compressor
  "grep short output (pass through)"
  grep-compress "grep foo file.txt"
  "file.txt:foo\nfile.txt:foobar"
  "file.txt:foo\nfile.txt:foobar")

(test-compressor
  "rg non-grep command"
  grep-compress "git log"
  "commit abc123"
  nil)

# ── find ──
(test-compressor
  "find short output"
  find-compress "find . -name '*.rs'"
  "src/main.rs\nsrc/lib.rs"
  "src/main.rs\nsrc/lib.rs")

(test-compressor
  "find non-find command"
  find-compress "ls -la"
  "total 12"
  nil)

# ── ls ──
(test-compressor
  "ls short output"
  ls-compress "ls -la"
  "total 12\ndrwxr-xr-x 2 user user 4096 Jan 1 ."
  "total 12\ndrwxr-xr-x 2 user user 4096 Jan 1 .")

(test-compressor
  "ls non-ls command"
  ls-compress "find ."
  "file1\nfile2"
  nil)

# ── curl ──
(test-compressor
  "curl short output"
  curl-compress "curl https://example.com"
  "hello"
  "hello")

(test-compressor
  "curl non-curl command"
  curl-compress "wget https://example.com"
  "hello"
  nil)

# ── npm ──
(test-compressor
  "npm install"
  npm-compress "npm install express"
  "added 50 packages from 40 contributors in 3s"
  "ok (50 deps)")

(test-compressor
  "npm non-npm command"
  npm-compress "git clone url"
  "Cloning..."
  nil)

# ── git ──
(test-compressor
  "git status clean"
  git-compress "git status"
  "On branch main\nnothing to commit, working tree clean"
  "main\nclean")

(test-compressor
  "git non-git command"
  git-compress "cargo build"
  "Compiling..."
  nil)

# ── cargo ──
(test-compressor
  "cargo build success"
  cargo-compress "cargo build"
  "   Compiling dirge v0.1.0\n    Finished dev [unoptimized] target(s) in 12.34s"
  "compiled 1 crates\n(12.34s)")

(test-compressor
  "cargo non-cargo command"
  cargo-compress "git status"
  "On branch main"
  nil)

# ── docker ──
(test-compressor
  "docker ps"
  docker-compress "docker ps"
  "CONTAINER ID   IMAGE          COMMAND                  CREATED        STATUS        PORTS                    NAMES\nabc123         nginx:latest   /docker-entrypoint.sh    2 hours ago    Up 2 hours    0.0.0.0:8080->80/tcp     webserver"
  "webserver (nginx:latest): Up 2 hours")

(test-compressor
  "docker non-docker command"
  docker-compress "git log"
  "commit abc123"
  nil)

# ── kubectl ──
(test-compressor
  "kubectl get pods"
  kubectl-compress "kubectl get pods"
  "NAME   READY   STATUS    RESTARTS   AGE\npod1   1/1     Running   0          2d\npod2   1/1     Running   0          2d"
  "[NAME READY STATUS RESTARTS]\npod1 1/1 Running 0 2d\npod2 1/1 Running 0 2d")

(test-compressor
  "kubectl non-kubectl command"
  kubectl-compress "docker ps"
  "CONTAINER ID..."
  nil)

# ── Tool gate: on-tool-end only processes shell tools ──
(var gate-test-result nil)
(set harness-replace-result nil)
(on-tool-end @{:tool "read" :output "some file content" :command ""})
(assert-str "read tool skips compression" nil harness-replace-result)

(set harness-replace-result nil)
(on-tool-end @{:tool "bash" :output "On branch main\nnothing to commit" :command "git status"})
(def gate-got harness-replace-result)
(assert-str "bash tool compresses git output" "main\nclean" gate-got)

(set harness-replace-result nil)
(var ls-lines @[])
(for i 1 51 (array/push ls-lines "line"))
(on-tool-end @{:tool "bash_output" :output (string/join ls-lines "\n") :command "ls -la"})
(assert-str "bash_output tool compresses large ls output" "50 entries (50 files, 0 dirs)" harness-replace-result)

(print "\n--- All tests completed ---")