vllm-bench.md

date	backend	model_id	tokenizer_id	num_prompts	framework	request_rate	burstiness	max_concurrency	duration	completed	total_input_tokens	total_output_tokens	request_throughput	request_goodput:	output_throughput	total_token_throughput	mean_ttft_ms	median_ttft_ms	std_ttft_ms	p99_ttft_ms	mean_tpot_ms	median_tpot_ms	std_tpot_ms	p99_tpot_ms	mean_itl_ms	median_itl_ms	std_itl_ms	p99_itl_ms
2025-04-10 01:08:42	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	120	vllm	1	1	nan	102.546	120	120000	12000	1.1702	nan	117.02	1287.22	418.944	358.51	117.766	750.236	56.8231	54.1962	11.0061	84.6333	56.8231	37.1144	79.7653	320.912
2025-04-10 01:16:56	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	1200	vllm	10	1	nan	483.386	1200	1200000	120000	2.48249	nan	248.249	2730.74	174863	175294	102874	350122	100.18	77.0298	228.431	627.818	100.166	43.5764	2296.79	466.211
2025-04-10 01:25:57	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	2400	vllm	20	1	nan	529.077	1308	1308000	130800	2.47223	nan	247.223	2719.45	223113	228206	126454	401587	106.944	79.7957	286.196	895.216	106.93	43.643	2870.69	325.799
2025-04-10 01:34:58	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	3600	vllm	30	1	nan	528.253	1308	1308000	130800	2.47609	nan	247.609	2723.7	229363	236590	128320	401989	110.216	77.4306	316.987	1212.2	110.202	43.5953	3168.12	317.885
2025-04-10 01:43:58	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	4200	vllm	35	1	nan	527.925	1308	1308000	130800	2.47763	nan	247.763	2725.39	232598	241346	129180	401945	106.615	77.1056	292.932	1074.84	106.601	43.6504	2932.13	317.435
2025-04-10 01:51:01	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	2000	vllm	inf	1	nan	411.181	1017	1017000	101700	2.47337	nan	247.337	2720.7	201785	202555	116776	401101	103.271	76.7273	269.71	817.872	103.259	43.47	2703.82	174.292
2025-04-10 02:34:40	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	120	sgl	1	1	nan	102.572	120	120000	12000	1.16991	nan	116.991	1286.9	414.314	355.465	118.829	767.049	56.7146	54.2361	10.9518	84.6905	56.7146	37.1214	77.1924	320.552
2025-04-10 02:42:55	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	1200	sgl	10	1	nan	483.417	1200	1200000	120000	2.48233	nan	248.233	2730.56	174895	175335	102881	350161	100.205	77.0311	228.438	627.861	100.192	43.5786	2296.88	466.479
2025-04-10 02:49:21	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	2400	sgl	20	1	nan	375.05	902	902000	90200	2.40501	nan	240.501	2645.51	157595	157513	92251.6	312789	82.4376	77.8888	88.1399	116.955	82.4358	43.6637	936.806	325.905
2025-04-10 03:06:31	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	120	sgl	1	1	nan	102.575	120	120000	12000	1.16987	nan	116.987	1286.86	411.621	351.711	117.848	760.864	56.6146	54.1877	10.958	84.8155	56.6146	37.086	76.7083	321.25
2025-04-10 03:14:46	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	1200	sgl	10	1	nan	483.418	1200	1200000	120000	2.48232	nan	248.232	2730.56	174889	175324	102880	350160	100.202	77.0516	228.439	627.851	100.189	43.5777	2296.87	466.509
2025-04-10 03:23:47	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	2400	sgl	20	1	nan	529.097	1308	1308000	130800	2.47214	nan	247.214	2719.35	223124	228232	126451	401526	106.957	79.7714	286.196	895.276	106.943	43.6914	2870.68	325.283
2025-04-10 03:32:47	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	3600	sgl	30	1	nan	528.263	1308	1308000	130800	2.47604	nan	247.604	2723.65	229370	236604	128318	401994	110.222	77.2302	316.996	1214.58	110.208	43.6342	3168.11	318.624
2025-04-10 03:41:48	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	4200	sgl	35	1	nan	527.982	1308	1308000	130800	2.47736	nan	247.736	2725.09	232607	241355	129183	401978	106.643	77.1012	292.979	1075.34	106.629	43.6526	2932.5	318.157
2025-04-10 03:48:50	vllm	meta-llama/Llama-3.1-8B-Instruct	meta-llama/Llama-3.1-8B-Instruct	2000	sgl	inf	1	nan	411.147	1017	1017000	101700	2.47357	nan	247.357	2720.93	201752	202539	116795	401080	103.388	76.9119	269.689	817.844	103.375	43.5	2703.83	175.695