Skip to content

Instantly share code, notes, and snippets.

@nateraw

nateraw/bench.sh

Created July 24, 2026 02:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save nateraw/5ac359374bf2810770f1a57d891f9794 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save nateraw/5ac359374bf2810770f1a57d891f9794 to your computer and use it in GitHub Desktop.
Download ZIP
poolside/Laguna-S-2.1-NVFP4 - Single DGX Spark Benchmark
Raw
bench.sh
MODEL_NAME="poolside/Laguna-S-2.1-NVFP4"
source .venv/bin/activate
uvx llama-benchy \
--base-url http://localhost:8000/v1 \
--model "$MODEL_NAME" \
--depth 0 4096 8192 16384 32768 65535 100000 \
--pp 2048 \
--tg 128 \
--enable-prefix-caching \
--concurrency 1 2 5 10 \
--save-result results.csv
Raw
results.csv
We can make this file beautiful and searchable if this error is corrected: No commas found in this CSV file in line 0.
| model | test | t/s (total) | t/s (req) | peak t/s | peak t/s (req) | ttfr (ms) | est_ppt (ms) | e2e_ttft (ms) |
|:----------------------------|-----------------------:|----------------:|-----------------:|---------------:|-----------------:|----------------------:|----------------------:|----------------------:|
| poolside/Laguna-S-2.1-NVFP4 | pp2048 (c1) | 1945.70 ± 60.45 | 1945.70 ± 60.45 | | | 1056.97 ± 33.48 | 1054.13 ± 33.48 | 1056.97 ± 33.48 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 (c1) | 19.36 ± 1.26 | 19.36 ± 1.26 | 27.00 ± 0.82 | 27.00 ± 0.82 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 (c2) | 2329.02 ± 7.96 | 1216.94 ± 50.62 | | | 1689.35 ± 70.03 | 1686.50 ± 70.03 | 1689.35 ± 70.03 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 (c2) | 28.18 ± 1.03 | 14.94 ± 0.93 | 44.67 ± 4.99 | 24.83 ± 5.15 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 (c5) | 2450.62 ± 0.54 | 740.19 ± 243.54 | | | 3063.41 ± 915.36 | 3060.56 ± 915.36 | 3063.41 ± 915.36 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 (c5) | 40.37 ± 0.65 | 9.98 ± 0.77 | 64.33 ± 2.87 | 18.33 ± 2.80 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 (c10) | 2430.47 ± 15.71 | 504.94 ± 278.17 | | | 5174.88 ± 2202.03 | 5172.04 ± 2202.03 | 5174.88 ± 2202.03 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 (c10) | 56.24 ± 0.24 | 7.62 ± 0.86 | 110.33 ± 5.79 | 17.30 ± 3.20 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d4096 (c1) | 2349.14 ± 20.23 | 2349.14 ± 20.23 | | | 1747.44 ± 15.11 | 1744.60 ± 15.11 | 1747.44 ± 15.11 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d4096 (c1) | 18.27 ± 1.57 | 18.27 ± 1.57 | 25.67 ± 0.47 | 25.67 ± 0.47 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d4096 (c1) | 1890.05 ± 8.41 | 1890.05 ± 8.41 | | | 1086.43 ± 4.81 | 1083.59 ± 4.81 | 1086.43 ± 4.81 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d4096 (c1) | 19.81 ± 1.22 | 19.81 ± 1.22 | 28.67 ± 3.40 | 28.67 ± 3.40 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d4096 (c2) | 2390.08 ± 4.94 | 1458.15 ± 262.03 | | | 2906.94 ± 521.99 | 2904.09 ± 521.99 | 2906.94 ± 521.99 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d4096 (c2) | 27.94 ± 0.90 | 15.58 ± 0.90 | 46.67 ± 2.05 | 26.17 ± 2.91 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d4096 (c2) | 2223.66 ± 12.55 | 1159.39 ± 46.24 | | | 1772.11 ± 70.58 | 1769.26 ± 70.58 | 1772.11 ± 70.58 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d4096 (c2) | 28.04 ± 0.55 | 15.73 ± 2.74 | 53.33 ± 11.67 | 31.17 ± 11.61 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d4096 (c5) | 2478.80 ± 2.71 | 904.38 ± 439.60 | | | 5487.02 ± 2095.09 | 5484.17 ± 2095.09 | 5487.02 ± 2095.09 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d4096 (c5) | 32.77 ± 1.87 | 9.35 ± 1.44 | 74.00 ± 9.42 | 19.07 ± 2.67 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d4096 (c5) | 2309.26 ± 36.34 | 699.87 ± 232.33 | | | 3244.94 ± 979.54 | 3242.10 ± 979.54 | 3244.94 ± 979.54 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d4096 (c5) | 40.08 ± 0.31 | 9.98 ± 0.93 | 70.67 ± 1.89 | 18.67 ± 3.96 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d4096 (c10) | 2467.57 ± 8.63 | 598.20 ± 426.67 | | | 9661.02 ± 4578.95 | 9658.18 ± 4578.95 | 9661.02 ± 4578.95 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d4096 (c10) | 41.35 ± 1.76 | 7.29 ± 2.18 | 118.67 ± 13.30 | 19.23 ± 8.29 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d4096 (c10) | 2291.77 ± 19.65 | 479.92 ± 266.18 | | | 5479.33 ± 2385.31 | 5476.49 ± 2385.31 | 5479.33 ± 2385.31 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d4096 (c10) | 54.45 ± 2.34 | 7.35 ± 1.21 | 107.33 ± 11.03 | 16.70 ± 5.37 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d8192 (c1) | 2390.39 ± 6.89 | 2390.39 ± 6.89 | | | 3430.49 ± 10.21 | 3427.65 ± 10.21 | 3430.49 ± 10.21 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d8192 (c1) | 18.21 ± 0.29 | 18.21 ± 0.29 | 26.67 ± 0.47 | 26.67 ± 0.47 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d8192 (c1) | 1800.56 ± 4.20 | 1800.56 ± 4.20 | | | 1140.27 ± 2.65 | 1137.43 ± 2.65 | 1140.27 ± 2.65 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d8192 (c1) | 20.44 ± 1.21 | 20.44 ± 1.21 | 29.67 ± 1.70 | 29.67 ± 1.70 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d8192 (c2) | 2473.18 ± 11.04 | 1579.81 ± 342.89 | | | 5445.71 ± 1180.86 | 5442.86 ± 1180.86 | 5445.71 ± 1180.86 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d8192 (c2) | 19.06 ± 1.65 | 12.11 ± 1.85 | 34.67 ± 1.70 | 19.67 ± 2.36 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d8192 (c2) | 2140.89 ± 1.93 | 1114.24 ± 42.11 | | | 1843.50 ± 69.56 | 1840.65 ± 69.56 | 1843.50 ± 69.56 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d8192 (c2) | 26.75 ± 2.20 | 14.19 ± 1.13 | 41.00 ± 2.16 | 23.33 ± 3.59 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d8192 (c5) | 2481.85 ± 12.88 | 986.81 ± 514.75 | | | 10407.21 ± 4342.41 | 10404.36 ± 4342.41 | 10407.21 ± 4342.41 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d8192 (c5) | 23.43 ± 1.72 | 7.91 ± 1.84 | 60.67 ± 2.62 | 17.60 ± 3.18 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d8192 (c5) | 2222.17 ± 24.44 | 671.61 ± 220.21 | | | 3374.91 ± 1012.26 | 3372.06 ± 1012.26 | 3374.91 ± 1012.26 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d8192 (c5) | 39.28 ± 0.49 | 9.49 ± 0.75 | 66.33 ± 0.47 | 19.60 ± 3.98 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d8192 (c10) | 2439.96 ± 5.92 | 647.10 ± 497.76 | | | 18774.07 ± 9413.99 | 18771.22 ± 9413.99 | 18774.07 ± 9413.99 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d8192 (c10) | 27.69 ± 0.53 | 6.32 ± 3.67 | 109.67 ± 13.89 | 16.63 ± 10.03 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d8192 (c10) | 2231.90 ± 2.34 | 468.46 ± 261.65 | | | 5627.13 ± 2453.91 | 5624.29 ± 2453.91 | 5627.13 ± 2453.91 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d8192 (c10) | 54.37 ± 0.90 | 7.35 ± 1.04 | 111.33 ± 7.32 | 17.37 ± 6.04 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d16384 (c1) | 2409.56 ± 1.00 | 2409.56 ± 1.00 | | | 6802.98 ± 2.88 | 6800.14 ± 2.88 | 6802.98 ± 2.88 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d16384 (c1) | 18.18 ± 0.87 | 18.18 ± 0.87 | 24.33 ± 2.05 | 24.33 ± 2.05 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d16384 (c1) | 1687.19 ± 8.75 | 1687.19 ± 8.75 | | | 1216.73 ± 6.29 | 1213.89 ± 6.29 | 1216.73 ± 6.29 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d16384 (c1) | 16.50 ± 0.26 | 16.50 ± 0.26 | 25.00 ± 2.16 | 25.00 ± 2.16 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d16384 (c2) | 2443.46 ± 8.94 | 1683.55 ± 461.71 | | | 10527.14 ± 2885.56 | 10524.30 ± 2885.56 | 10527.14 ± 2885.56 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d16384 (c2) | 18.35 ± 1.14 | 13.05 ± 4.00 | 48.67 ± 15.06 | 28.67 ± 12.83 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d16384 (c2) | 1991.75 ± 5.01 | 1032.52 ± 34.83 | | | 1988.60 ± 66.99 | 1985.75 ± 66.99 | 1989.24 ± 67.10 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d16384 (c2) | 22.69 ± 1.50 | 12.30 ± 1.35 | 32.00 ± 3.74 | 17.83 ± 1.86 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d16384 (c5) | 2418.98 ± 2.36 | 1033.76 ± 601.31 | | | 20820.35 ± 9318.98 | 20817.51 ± 9318.98 | 20820.79 ± 9319.35 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d16384 (c5) | 16.41 ± 0.48 | 7.76 ± 3.51 | 52.67 ± 3.09 | 17.73 ± 5.64 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d16384 (c5) | 2085.98 ± 2.21 | 629.32 ± 206.37 | | | 3601.05 ± 1078.22 | 3598.20 ± 1078.22 | 3601.15 ± 1078.34 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d16384 (c5) | 36.65 ± 1.43 | 9.62 ± 1.99 | 75.00 ± 12.25 | 19.93 ± 7.05 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d16384 (c10) | 2381.14 ± 9.13 | 672.86 ± 568.31 | | | 38057.28 ± 19691.16 | 38054.43 ± 19691.16 | 38057.68 ± 19691.17 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d16384 (c10) | 16.35 ± 1.46 | 4.98 ± 2.71 | 57.33 ± 11.84 | 13.14 ± 5.15 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d16384 (c10) | 2054.52 ± 11.89 | 428.23 ± 236.76 | | | 6128.38 ± 2656.39 | 6125.54 ± 2656.39 | 6129.08 ± 2656.64 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d16384 (c10) | 48.83 ± 2.64 | 6.87 ± 1.23 | 105.67 ± 13.89 | 16.17 ± 5.63 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d32768 (c1) | 2294.24 ± 9.52 | 2294.24 ± 9.52 | | | 14286.53 ± 59.36 | 14283.68 ± 59.36 | 14290.19 ± 59.30 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d32768 (c1) | 20.56 ± 0.71 | 20.56 ± 0.71 | 29.33 ± 1.89 | 29.33 ± 1.89 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d32768 (c1) | 1474.15 ± 1.04 | 1474.15 ± 1.04 | | | 1392.12 ± 0.98 | 1389.27 ± 0.98 | 1394.84 ± 2.90 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d32768 (c1) | 19.17 ± 1.78 | 19.17 ± 1.78 | 27.00 ± 2.94 | 27.00 ± 2.94 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d32768 (c2) | 2280.21 ± 1.42 | 1661.95 ± 521.63 | | | 21875.08 ± 6864.50 | 21872.23 ± 6864.50 | 21877.59 ± 6865.00 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d32768 (c2) | 10.82 ± 0.08 | 9.73 ± 3.32 | 31.67 ± 0.94 | 21.50 ± 2.06 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d32768 (c2) | 1700.59 ± 44.25 | 879.32 ± 36.19 | | | 2335.90 ± 96.62 | 2333.05 ± 96.62 | 2338.32 ± 95.34 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d32768 (c2) | 25.50 ± 0.88 | 13.04 ± 0.43 | 35.00 ± 2.83 | 19.50 ± 0.96 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d32768 (c5) | 2259.59 ± 1.93 | 1011.76 ± 629.83 | | | 43894.06 ± 20398.26 | 43891.21 ± 20398.26 | 43896.07 ± 20398.41 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d32768 (c5) | 9.04 ± 0.09 | 5.64 ± 3.49 | 37.67 ± 2.36 | 15.80 ± 6.33 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d32768 (c5) | 1822.73 ± 2.63 | 547.13 ± 178.12 | | | 4135.82 ± 1230.42 | 4132.97 ± 1230.42 | 4136.14 ± 1230.07 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d32768 (c5) | 34.91 ± 1.81 | 9.05 ± 2.17 | 67.00 ± 8.04 | 18.67 ± 5.94 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d32768 (c10) | 2252.99 ± 7.41 | 651.12 ± 574.36 | | | 80270.31 ± 41751.87 | 80267.47 ± 41751.87 | 80271.02 ± 41750.93 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d32768 (c10) | 8.23 ± 0.53 | 4.58 ± 3.68 | 42.33 ± 6.60 | 12.49 ± 7.40 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d32768 (c10) | 455.12 ± 238.33 | 118.96 ± 174.51 | | | 30984.92 ± 16000.73 | 30982.07 ± 16000.73 | 30985.14 ± 16000.34 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d32768 (c10) | 22.79 ± 5.08 | 4.77 ± 2.41 | 73.67 ± 5.73 | 12.87 ± 3.76 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d65535 (c1) | 2036.61 ± 0.98 | 2036.61 ± 0.98 | | | 32182.35 ± 15.54 | 32179.51 ± 15.54 | 32182.35 ± 15.54 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d65535 (c1) | 21.13 ± 1.99 | 21.13 ± 1.99 | 30.00 ± 4.32 | 30.00 ± 4.32 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d65535 (c1) | 1183.18 ± 3.51 | 1183.18 ± 3.51 | | | 1733.79 ± 5.13 | 1730.94 ± 5.13 | 1737.68 ± 7.33 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d65535 (c1) | 16.96 ± 2.21 | 16.96 ± 2.21 | 26.33 ± 1.25 | 26.33 ± 1.25 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d65535 (c2) | 2008.88 ± 2.54 | 1503.95 ± 499.46 | | | 48980.98 ± 16265.70 | 48978.14 ± 16265.70 | 48982.83 ± 16263.85 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d65535 (c2) | 6.04 ± 0.29 | 10.41 ± 6.87 | 31.33 ± 7.13 | 22.17 ± 5.87 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d65535 (c2) | 1387.00 ± 5.14 | 711.54 ± 17.34 | | | 2882.82 ± 70.16 | 2879.98 ± 70.16 | 2882.82 ± 70.16 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d65535 (c2) | 25.88 ± 2.21 | 14.67 ± 1.79 | 40.67 ± 6.65 | 23.67 ± 2.92 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d65535 (c5) | 1994.69 ± 4.54 | 910.49 ± 581.61 | | | 98801.62 ± 46555.04 | 98798.78 ± 46555.04 | 98803.51 ± 46554.61 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d65535 (c5) | 3.94 ± 0.25 | 5.22 ± 4.75 | 26.00 ± 7.12 | 10.73 ± 5.56 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d65535 (c5) | 1441.78 ± 1.70 | 428.58 ± 137.47 | | | 5263.36 ± 1542.75 | 5260.52 ± 1542.75 | 5263.69 ± 1542.40 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d65535 (c5) | 30.75 ± 0.82 | 7.63 ± 0.94 | 54.33 ± 3.40 | 15.27 ± 1.98 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d65535 (c10) | 1981.39 ± 1.49 | 581.71 ± 525.55 | | | 182102.92 ± 95193.75 | 182100.08 ± 95193.75 | 182103.67 ± 95192.89 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d65535 (c10) | 3.81 ± 0.02 | 3.80 ± 3.34 | 32.00 ± 4.24 | 10.81 ± 6.23 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d65535 (c10) | 66.15 ± 5.65 | 46.25 ± 143.68 | | | 162467.03 ± 95635.75 | 162464.18 ± 95635.75 | 162467.13 ± 95635.61 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d65535 (c10) | 4.23 ± 0.17 | 3.96 ± 3.78 | 33.33 ± 1.70 | 11.40 ± 6.84 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d100000 (c1) | 1797.01 ± 0.45 | 1797.01 ± 0.45 | | | 55652.04 ± 13.82 | 55649.19 ± 13.82 | 55653.97 ± 11.53 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d100000 (c1) | 16.06 ± 0.77 | 16.06 ± 0.77 | 26.00 ± 0.82 | 26.00 ± 0.82 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d100000 (c1) | 993.12 ± 2.92 | 993.12 ± 2.92 | | | 2065.05 ± 6.06 | 2062.21 ± 6.06 | 2069.98 ± 9.54 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d100000 (c1) | 16.81 ± 2.27 | 16.81 ± 2.27 | 34.33 ± 13.42 | 34.33 ± 13.42 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d100000 (c2) | 1771.24 ± 3.23 | 1329.29 ± 443.61 | | | 84660.22 ± 28251.59 | 84657.37 ± 28251.59 | 84663.45 ± 28253.76 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d100000 (c2) | 3.92 ± 0.19 | 11.39 ± 9.22 | 38.33 ± 16.05 | 22.33 ± 16.89 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d100000 (c2) | 1133.56 ± 10.73 | 579.91 ± 13.48 | | | 3536.37 ± 82.40 | 3533.52 ± 82.40 | 3537.53 ± 81.82 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d100000 (c2) | 22.37 ± 2.32 | 12.20 ± 2.01 | 34.67 ± 4.50 | 20.50 ± 4.65 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d100000 (c5) | 1758.60 ± 10.58 | 806.28 ± 515.47 | | | 170523.90 ± 80679.31 | 170521.05 ± 80679.31 | 170524.45 ± 80679.49 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d100000 (c5) | 2.27 ± 0.26 | 5.75 ± 6.75 | 29.00 ± 4.55 | 12.62 ± 8.96 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d100000 (c5) | 47.76 ± 6.31 | 21.83 ± 10.79 | | | 123085.23 ± 63080.66 | 123082.39 ± 63080.66 | 123086.28 ± 63079.59 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d100000 (c5) | 3.71 ± 0.84 | 5.22 ± 5.27 | 25.67 ± 0.94 | 11.67 ± 6.21 | | | |
| poolside/Laguna-S-2.1-NVFP4 | ctx_pp @ d100000 (c10) | 1745.99 ± 2.62 | 514.86 ± 466.60 | | | 314879.35 ± 164997.40 | 314876.51 ± 164997.40 | 314879.71 ± 164997.10 |
| poolside/Laguna-S-2.1-NVFP4 | ctx_tg @ d100000 (c10) | 2.19 ± 0.07 | 3.80 ± 4.61 | 28.67 ± 6.65 | 9.31 ± 6.88 | | | |
| poolside/Laguna-S-2.1-NVFP4 | pp2048 @ d100000 (c10) | 35.82 ± 1.67 | 33.79 ± 126.54 | | | 306536.12 ± 172262.86 | 306533.27 ± 172262.86 | 306536.47 ± 172262.82 |
| poolside/Laguna-S-2.1-NVFP4 | tg128 @ d100000 (c10) | 2.30 ± 0.04 | 3.90 ± 4.70 | 30.33 ± 8.50 | 9.83 ± 7.72 | | | |
Raw
serve.sh
export CUTE_DSL_ARCH=sm_121a # arch string for FP4 kernel JIT
export PATH=/usr/local/cuda/bin:$PATH # nvcc for JIT
export MAX_JOBS=4 # cap JIT fan-out; see warning below
source .venv/bin/activate
vllm serve poolside/Laguna-S-2.1-NVFP4 \
--revision b482b5d57fda6e4e562a652869bde24ba2a57c92 \
--speculative-config '{"model":"poolside/Laguna-S-2.1-DFlash-NVFP4","num_speculative_tokens":15,"revision":"723794750422b3efbf3a7b3af76dffb4ba035943"}' \
--enable-auto-tool-choice \
--tool-call-parser poolside_v1 \
--reasoning-parser poolside_v1 \
--override-generation-config '{"temperature":0.7,"top_p":0.95}' \
--max-num-seqs 32 \
--max-model-len 262144 \
--gpu-memory-utilization 0.85 \
--host 0.0.0.0 \
--port 8000
# --load-format dummy
Raw
setup.sh
# Python headers: Triton JIT needs them and DGX OS ships without them.
# Without this, the first server start dies in triton/runtime/build.py.
sudo apt install -y python3.12-dev
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv -p 3.12
# vLLM 0.25.1 with CUDA-13 torch (aarch64 wheels are on PyPI)
uv pip install -p .venv/ vllm==0.25.1 --torch-backend=cu130
# FlashInfer nightly trio: without flashinfer-python the NVFP4 path is not
# native; the jit-cache wheel avoids most first-start JIT compilation.
uv pip install -p .venv/ \
"flashinfer-python==0.6.15.dev20260712" \
"flashinfer-cubin==0.6.15.dev20260712" \
"flashinfer-jit-cache==0.6.15.dev20260712" \
--extra-index-url https://flashinfer.ai/whl/nightly/ \
--extra-index-url https://flashinfer.ai/whl/nightly/cu130/ \
--index-strategy unsafe-best-match
hf download poolside/Laguna-S-2.1-NVFP4 --revision b482b5d57fda6e4e562a652869bde24ba2a57c92
hf download poolside/Laguna-S-2.1-DFlash-NVFP4 --revision 723794750422b3efbf3a7b3af76dffb4ba035943 # 73 GB total
@nateraw

nateraw commented Jul 24, 2026

Copy link
Copy Markdown
Author

Laguna-S-2.1-NVFP4

https://huggingface.co/poolside/Laguna-S-2.1-NVFP4

Setup (one-time)

./setup.sh

Serve

# I had to do this while ssh'd in, but idk if necessary 🤷‍♂️
# mkdir -p /home/<user>/.triton/
# mkdir -p /home/<user>/.cache/vllm/
# sudo chmod -R 777 /home/<user>/.triton/
# sudo chmod -R 777 /home/<user>/.cache/vllm/

./serve.sh

Benchmark

./bench.sh

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment