Last active
June 26, 2026 20:57
-
-
Save sshleifer/8c19b3465f284ef9cc183360deac42ba to your computer and use it in GitHub Desktop.
SGLang GLM-5.2 NVFP4 DP4 invalid probability repro artifacts
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py | |
| index f66ea9a672..586e326434 100644 | |
| --- a/python/sglang/srt/utils/hf_transformers/config.py | |
| +++ b/python/sglang/srt/utils/hf_transformers/config.py | |
| @@ -60,12 +60,33 @@ class HfModelConfigParser(ModelConfigParserBase): | |
| revision: Optional[str] = None, | |
| **kwargs, | |
| ): | |
| - config = AutoConfig.from_pretrained( | |
| - model, | |
| - trust_remote_code=trust_remote_code, | |
| - revision=revision, | |
| - **kwargs, | |
| - ) | |
| + try: | |
| + config = AutoConfig.from_pretrained( | |
| + model, | |
| + trust_remote_code=trust_remote_code, | |
| + revision=revision, | |
| + **kwargs, | |
| + ) | |
| + except Exception as exc: | |
| + if "deepseek_sparse_attention" not in str(exc): | |
| + raise | |
| + from transformers import PretrainedConfig | |
| + from transformers.models.glm_moe_dsa.configuration_glm_moe_dsa import ( | |
| + GlmMoeDsaConfig, | |
| + ) | |
| + | |
| + raw_config, _ = PretrainedConfig.get_config_dict( | |
| + model, revision=revision, **kwargs | |
| + ) | |
| + if raw_config.get("architectures") != ["GlmMoeDsaForCausalLM"]: | |
| + raise | |
| + layer_types = raw_config.get("layer_types") | |
| + if isinstance(layer_types, list): | |
| + raw_config["layer_types"] = [ | |
| + "sparse" if x == "deepseek_sparse_attention" else x | |
| + for x in layer_types | |
| + ] | |
| + config = GlmMoeDsaConfig.from_dict(raw_config) | |
| if ( | |
| config.architectures is not None |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| H4sIAAAAAAAAA+3PsU7bUBQG4BsE6RN0t7qBFHRtxw4sFQtMnVlRCIiFqMiuWZNnYEKdWNlh5VkQ | |
| PERH6iBQEUKVaAkS0vcN95xj/7LPHTV7w53R92p/rxkf7RyMjtKid7Cbx9iL6/mwV66WZVwrVtPB | |
| WlbEfL2M4fViq4wxTNt+cv7zYlbry5vPs+cxi/2ifZcWWTooikGa90PM0rSfheRf/vVqTf1jWCVJ | |
| qIfjv+bqw6YaN/V+Vb/HVu9msvlta6HTyT89zIvhKnReyP06Tr4+9hsPZ71ULc1/QwAAAAAAAAAA | |
| AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD6E | |
| L7fL3bYkj/N251ng+mRy15aFada9n89up7M5nIZZ8nn6rWz86Va6//OhJ/dL5rUsAAAAAMAc/Aaz | |
| v1LeABgBAA== |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import json | |
| import random | |
| import time | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| import httpx | |
| SENTENCES = [ | |
| "The regression involves a long lived request state machine, nested configuration, and a cache entry that must survive across turns.", | |
| "Assume every symbol name is intentionally similar to several nearby names so exact local context matters.", | |
| "Admission, scheduling, and accounting decisions must agree even when the prompt is very long.", | |
| "Several prior attempts fixed the happy path but changed behavior for legacy aliases and partially initialized objects.", | |
| "The answer should reason from evidence, preserve compatibility, and avoid broad rewrites that hide the real bug.", | |
| ] | |
| FOLLOWUPS = [ | |
| "The first patch still fails with mixed casing, nested configuration, and legacy compatibility paths. Re-check the invariant and propose a smaller fix.", | |
| "A focused regression test now fails only after a long context turn. Reason about request state, cache reuse, and decode behavior before answering.", | |
| "Give the final patch plan, edge cases, and a concise explanation of why the implementation should not regress adjacent behavior.", | |
| ] | |
| @dataclass | |
| class RequestResult: | |
| rollout_id: int | |
| turn: int | |
| ok: bool | |
| latency_s: float | |
| completed_s: float | |
| output_tokens: int = 0 | |
| prompt_tokens: int = 0 | |
| cached_tokens: int = 0 | |
| dp_rank: int | None = None | |
| error: str | None = None | |
| def endpoint(base_url: str, path: str) -> str: | |
| if base_url.endswith(path): | |
| return base_url | |
| return base_url.rstrip("/") + path | |
| def synthetic_words(kind: str, rollout_id: int, n_words: int) -> str: | |
| words: list[str] = [] | |
| section = 0 | |
| while len(words) < n_words: | |
| sentence = SENTENCES[(rollout_id + section) % len(SENTENCES)] | |
| pieces = [ | |
| f"[{kind}:{rollout_id}:{section}]", | |
| sentence, | |
| "Relevant pseudo-code:", | |
| f"def reconcile_state_{rollout_id}_{section}(request, cache, scheduler):", | |
| " assert request.trace_id in cache.index", | |
| " return scheduler.admit(request, cache.snapshot())", | |
| "Observed failure:", | |
| f"request_{rollout_id}_{section} loses bootstrap metadata after a long-context turn.", | |
| ] | |
| for piece in pieces: | |
| words.extend(piece.split()) | |
| section += 1 | |
| return " ".join(words[:n_words]) | |
| def make_prompt(rollout_id: int, prompt_words: int) -> str: | |
| return ( | |
| "You are debugging a synthetic long-context multi-turn serving issue. " | |
| "No external tools, repositories, sandboxes, tests, or file systems are available. " | |
| "Use only the prompt text below. Maintain exact details across turns and produce a " | |
| "patch-oriented analysis.\n\n" | |
| + synthetic_words("initial", rollout_id, prompt_words) | |
| + f"\n\nSynthetic task id: long-prompt-{rollout_id}\n" | |
| ) | |
| def make_followup(rollout_id: int, turn: int, followup_words: int) -> str: | |
| return ( | |
| FOLLOWUPS[(turn - 1) % len(FOLLOWUPS)] | |
| + "\n\nAdditional long-turn context:\n" | |
| + synthetic_words(f"followup-{turn}", rollout_id, followup_words) | |
| ) | |
| def extract_text(data: dict[str, Any]) -> str: | |
| text = data.get("text") | |
| if isinstance(text, str): | |
| return text | |
| if isinstance(text, list): | |
| return "\n".join(str(x) for x in text) | |
| return "" | |
| async def one_request( | |
| client: httpx.AsyncClient, | |
| base_url: str, | |
| prompt: str, | |
| output_len: int, | |
| temperature: float, | |
| rollout_id: int, | |
| turn: int, | |
| ) -> tuple[RequestResult, str]: | |
| payload = { | |
| "text": prompt, | |
| "sampling_params": { | |
| "temperature": temperature, | |
| "max_new_tokens": output_len, | |
| "ignore_eos": True, | |
| }, | |
| "stream": False, | |
| } | |
| t0 = time.perf_counter() | |
| try: | |
| resp = await client.post(endpoint(base_url, "/generate"), json=payload) | |
| latency_s = time.perf_counter() - t0 | |
| if resp.status_code >= 400: | |
| return ( | |
| RequestResult( | |
| rollout_id=rollout_id, | |
| turn=turn, | |
| ok=False, | |
| latency_s=latency_s, | |
| completed_s=0.0, | |
| error=resp.text[:500], | |
| ), | |
| "", | |
| ) | |
| data = resp.json() | |
| meta = data.get("meta_info", {}) | |
| return ( | |
| RequestResult( | |
| rollout_id=rollout_id, | |
| turn=turn, | |
| ok=True, | |
| latency_s=latency_s, | |
| completed_s=0.0, | |
| output_tokens=int(meta.get("completion_tokens") or 0), | |
| prompt_tokens=int(meta.get("prompt_tokens") or 0), | |
| cached_tokens=int(meta.get("cached_tokens") or 0), | |
| dp_rank=meta.get("dp_rank"), | |
| ), | |
| extract_text(data), | |
| ) | |
| except Exception as exc: | |
| return ( | |
| RequestResult( | |
| rollout_id=rollout_id, | |
| turn=turn, | |
| ok=False, | |
| latency_s=time.perf_counter() - t0, | |
| completed_s=0.0, | |
| error=repr(exc), | |
| ), | |
| "", | |
| ) | |
| async def run(args: argparse.Namespace) -> list[RequestResult]: | |
| started = time.time() | |
| next_rollout_id = 0 | |
| results: list[RequestResult] = [] | |
| lock = asyncio.Lock() | |
| rng = random.Random(args.seed) | |
| workers = min(args.client_workers, args.concurrency, args.num_examples) | |
| semaphore = asyncio.Semaphore(args.concurrency) | |
| async with httpx.AsyncClient( | |
| timeout=httpx.Timeout( | |
| connect=args.connect_timeout_s, | |
| read=args.request_timeout_s, | |
| write=120.0, | |
| pool=args.request_timeout_s, | |
| ), | |
| limits=httpx.Limits( | |
| max_connections=args.client_connections, | |
| max_keepalive_connections=args.client_connections, | |
| ), | |
| ) as client: | |
| async def rollout(rollout_id: int) -> None: | |
| prompt = make_prompt(rollout_id, args.prompt_words) | |
| last_text = "" | |
| for turn in range(args.num_turns): | |
| if turn > 0: | |
| prompt += "\n\nAssistant previous attempt:\n" + last_text[: args.keep_chars] | |
| prompt += "\n\nUser follow-up:\n" + make_followup( | |
| rollout_id, turn, args.followup_words | |
| ) | |
| if args.stall_jitter_s: | |
| await asyncio.sleep(rng.random() * args.stall_jitter_s) | |
| async with semaphore: | |
| rr, last_text = await one_request( | |
| client, | |
| args.base_url, | |
| prompt, | |
| args.output_len, | |
| args.temperature, | |
| rollout_id, | |
| turn, | |
| ) | |
| rr.completed_s = time.time() - started | |
| async with lock: | |
| results.append(rr) | |
| if len(results) % args.progress_every == 0: | |
| ok = sum(1 for r in results if r.ok) | |
| out_tokens = sum(r.output_tokens for r in results if r.ok) | |
| print( | |
| f"[progress] requests={len(results)} ok={ok} " | |
| f"out_tokens={out_tokens} elapsed={rr.completed_s:.1f}s", | |
| flush=True, | |
| ) | |
| async def worker() -> None: | |
| nonlocal next_rollout_id | |
| while True: | |
| async with lock: | |
| if next_rollout_id >= args.num_examples: | |
| return | |
| rollout_id = next_rollout_id | |
| next_rollout_id += 1 | |
| await rollout(rollout_id) | |
| await asyncio.gather(*(worker() for _ in range(workers))) | |
| return results | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--base-url", default="http://127.0.0.1:30000") | |
| parser.add_argument("--run-id", default=f"glm52-dp4-invalid-prob-{int(time.time())}") | |
| parser.add_argument("--num-examples", type=int, default=128) | |
| parser.add_argument("--num-turns", type=int, default=8) | |
| parser.add_argument("--concurrency", type=int, default=256) | |
| parser.add_argument("--client-workers", type=int, default=256) | |
| parser.add_argument("--client-connections", type=int, default=256) | |
| parser.add_argument("--prompt-words", type=int, default=12000) | |
| parser.add_argument("--followup-words", type=int, default=4000) | |
| parser.add_argument("--output-len", type=int, default=2048) | |
| parser.add_argument("--temperature", type=float, default=1.0) | |
| parser.add_argument("--keep-chars", type=int, default=6000) | |
| parser.add_argument("--stall-jitter-s", type=float, default=0.0) | |
| parser.add_argument("--connect-timeout-s", type=float, default=120.0) | |
| parser.add_argument("--request-timeout-s", type=float, default=1800.0) | |
| parser.add_argument("--progress-every", type=int, default=24) | |
| parser.add_argument("--seed", type=int, default=0) | |
| parser.add_argument("--out-dir", type=Path, default=Path("repro_artifacts")) | |
| args = parser.parse_args() | |
| start = time.time() | |
| results = asyncio.run(run(args)) | |
| end = time.time() | |
| ok = [r for r in results if r.ok] | |
| summary = { | |
| "run_id": args.run_id, | |
| "base_url": args.base_url, | |
| "wall_s": end - start, | |
| "config": vars(args) | {"out_dir": str(args.out_dir)}, | |
| "requests": len(results), | |
| "ok_requests": len(ok), | |
| "client_output_tokens": sum(r.output_tokens for r in ok), | |
| "client_prompt_tokens": sum(r.prompt_tokens for r in ok), | |
| "errors": [asdict(r) for r in results if not r.ok][:50], | |
| } | |
| args.out_dir.mkdir(parents=True, exist_ok=True) | |
| out_path = args.out_dir / f"{args.run_id}.json" | |
| out_path.write_text(json.dumps(summary, indent=2)) | |
| print("[summary] " + json.dumps(summary, indent=2), flush=True) | |
| print(f"[wrote] {out_path}", flush=True) | |
| if __name__ == "__main__": | |
| main() |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [progress] requests=24 ok=24 out_tokens=49152 elapsed=127.9s | |
| [progress] requests=48 ok=48 out_tokens=98304 elapsed=127.9s | |
| [progress] requests=72 ok=72 out_tokens=147456 elapsed=128.0s | |
| [progress] requests=96 ok=96 out_tokens=196608 elapsed=128.1s | |
| [progress] requests=120 ok=120 out_tokens=245760 elapsed=128.1s | |
| [progress] requests=144 ok=144 out_tokens=294912 elapsed=262.5s | |
| [progress] requests=168 ok=168 out_tokens=344064 elapsed=262.6s | |
| [progress] requests=192 ok=192 out_tokens=393216 elapsed=262.6s | |
| [progress] requests=216 ok=216 out_tokens=442368 elapsed=262.7s | |
| [progress] requests=240 ok=240 out_tokens=491520 elapsed=262.8s | |
| [progress] requests=264 ok=264 out_tokens=540672 elapsed=420.3s | |
| [progress] requests=288 ok=288 out_tokens=589824 elapsed=420.3s | |
| [progress] requests=312 ok=312 out_tokens=638976 elapsed=420.4s | |
| [progress] requests=336 ok=336 out_tokens=688128 elapsed=420.5s | |
| [progress] requests=360 ok=360 out_tokens=737280 elapsed=420.5s | |
| [progress] requests=384 ok=384 out_tokens=786432 elapsed=420.6s | |
| [progress] requests=408 ok=408 out_tokens=835584 elapsed=604.2s | |
| [progress] requests=432 ok=432 out_tokens=884736 elapsed=604.2s | |
| [progress] requests=456 ok=456 out_tokens=933888 elapsed=604.3s | |
| [progress] requests=480 ok=480 out_tokens=983040 elapsed=604.4s | |
| [progress] requests=504 ok=504 out_tokens=1032192 elapsed=604.5s | |
| [progress] requests=528 ok=528 out_tokens=1081344 elapsed=810.7s | |
| [progress] requests=552 ok=552 out_tokens=1130496 elapsed=810.7s | |
| [progress] requests=576 ok=576 out_tokens=1179648 elapsed=819.1s | |
| [progress] requests=600 ok=600 out_tokens=1228800 elapsed=819.1s | |
| [progress] requests=624 ok=624 out_tokens=1277952 elapsed=819.2s | |
| [progress] requests=648 ok=624 out_tokens=1277952 elapsed=837.2s | |
| [progress] requests=672 ok=624 out_tokens=1277952 elapsed=837.3s | |
| [progress] requests=696 ok=624 out_tokens=1277952 elapsed=837.3s | |
| [progress] requests=720 ok=624 out_tokens=1277952 elapsed=837.3s | |
| [progress] requests=744 ok=624 out_tokens=1277952 elapsed=837.4s | |
| [progress] requests=768 ok=624 out_tokens=1277952 elapsed=837.4s | |
| [progress] requests=792 ok=624 out_tokens=1277952 elapsed=837.5s | |
| [progress] requests=816 ok=624 out_tokens=1277952 elapsed=837.5s | |
| [progress] requests=840 ok=624 out_tokens=1277952 elapsed=837.5s | |
| [progress] requests=864 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=888 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=912 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=936 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=960 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=984 ok=624 out_tokens=1277952 elapsed=837.6s | |
| [progress] requests=1008 ok=624 out_tokens=1277952 elapsed=837.7s | |
| [summary] { | |
| "run_id": "upstream-glm52-dp4-fake-c256-t8-1782504221", | |
| "base_url": "http://127.0.0.1:30000", | |
| "wall_s": 837.6550970077515, | |
| "config": { | |
| "base_url": "http://127.0.0.1:30000", | |
| "run_id": "upstream-glm52-dp4-fake-c256-t8-1782504221", | |
| "num_examples": 128, | |
| "num_turns": 8, | |
| "concurrency": 256, | |
| "client_workers": 256, | |
| "client_connections": 256, | |
| "prompt_words": 12000, | |
| "followup_words": 4000, | |
| "output_len": 2048, | |
| "temperature": 1.0, | |
| "keep_chars": 6000, | |
| "stall_jitter_s": 0.0, | |
| "connect_timeout_s": 120.0, | |
| "request_timeout_s": 1800.0, | |
| "progress_every": 24, | |
| "seed": 0, | |
| "out_dir": "repro_artifacts" | |
| }, | |
| "requests": 1024, | |
| "ok_requests": 624, | |
| "client_output_tokens": 1277952, | |
| "client_prompt_tokens": 25466328, | |
| "errors": [ | |
| { | |
| "rollout_id": 22, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.82943348400295, | |
| "completed_s": 837.1899147033691, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 80, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.89829513803124, | |
| "completed_s": 837.1930646896362, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 35, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.78840356785804, | |
| "completed_s": 837.1945824623108, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 43, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.8815170568414, | |
| "completed_s": 837.1960394382477, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 29, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.0610851990059, | |
| "completed_s": 837.1975336074829, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 14, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.72280930401757, | |
| "completed_s": 837.1989991664886, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 104, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.71563231991604, | |
| "completed_s": 837.2003016471863, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 95, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.0197383519262, | |
| "completed_s": 837.201637506485, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 81, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.7096065483056, | |
| "completed_s": 837.2031235694885, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 112, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.6929130749777, | |
| "completed_s": 837.204648733139, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 37, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.06073470506817, | |
| "completed_s": 837.2084038257599, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 55, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.91192385926843, | |
| "completed_s": 837.2098801136017, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 23, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.138181263115257, | |
| "completed_s": 837.2112166881561, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 103, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.54900413705036, | |
| "completed_s": 837.2127568721771, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 88, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.51140665076673, | |
| "completed_s": 837.2143564224243, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 98, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.078041835222393, | |
| "completed_s": 837.2158858776093, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 21, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.081916810944676, | |
| "completed_s": 837.2174029350281, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 118, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.506144625134766, | |
| "completed_s": 837.2188055515289, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 3, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.48803795175627, | |
| "completed_s": 837.2203915119171, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 119, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.07892961986363, | |
| "completed_s": 837.2219126224518, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 109, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.070175853092223, | |
| "completed_s": 837.223450422287, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 114, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.557316205929965, | |
| "completed_s": 837.2250330448151, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 110, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.578243990894407, | |
| "completed_s": 837.226574420929, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 26, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.165874192025512, | |
| "completed_s": 837.228175163269, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 113, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.169337579980493, | |
| "completed_s": 837.2297103404999, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 54, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.57338993670419, | |
| "completed_s": 837.2313106060028, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 116, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.512873705942184, | |
| "completed_s": 837.2329008579254, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 97, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.126189841888845, | |
| "completed_s": 837.2345006465912, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 77, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.073825867846608, | |
| "completed_s": 837.236025094986, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 79, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.56572635518387, | |
| "completed_s": 837.2376079559326, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 107, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.459730751346797, | |
| "completed_s": 837.239196062088, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 93, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.11286330409348, | |
| "completed_s": 837.2434024810791, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 125, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.111918911803514, | |
| "completed_s": 837.2449443340302, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 73, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.531345599796623, | |
| "completed_s": 837.2465291023254, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 30, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.547661412041634, | |
| "completed_s": 837.2480301856995, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 66, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.067817950155586, | |
| "completed_s": 837.2502574920654, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 78, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.174838203936815, | |
| "completed_s": 837.2517952919006, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 19, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.499992900993675, | |
| "completed_s": 837.2532937526703, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 38, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.49036482302472, | |
| "completed_s": 837.2553112506866, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 48, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.100534567143768, | |
| "completed_s": 837.2566833496094, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 65, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.179540033917874, | |
| "completed_s": 837.258579492569, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 31, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.504200456198305, | |
| "completed_s": 837.2606425285339, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 69, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.61858252901584, | |
| "completed_s": 837.262179851532, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 89, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.17133001703769, | |
| "completed_s": 837.2638218402863, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 87, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.163838885258883, | |
| "completed_s": 837.2653565406799, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 86, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.51901379181072, | |
| "completed_s": 837.266845703125, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 72, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.50911992089823, | |
| "completed_s": 837.2684087753296, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 60, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.14088002499193, | |
| "completed_s": 837.269923210144, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 63, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.218183803837746, | |
| "completed_s": 837.2714688777924, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 17, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.58583871414885, | |
| "completed_s": 837.2728791236877, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| } | |
| ] | |
| } | |
| [wrote] repro_artifacts/upstream-glm52-dp4-fake-c256-t8-1782504221.json |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /work/sam/sglang_upstream/python/sglang/launch_server.py:55: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. | |
| Example: sglang serve --model-path <model> [options] | |
| warnings.warn( | |
| [2026-06-26 19:58:56] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:58:56] Set dense attention kv len threshold to model index_topk=2048 for DeepSeek with DSA. | |
| [2026-06-26 19:58:56] Setting page size to 64 for DeepSeek DSA. | |
| [2026-06-26 19:58:56] Set DSA backends for fp8_e4m3 KV Cache: prefill=trtllm, decode=trtllm. | |
| [2026-06-26 19:58:56] Use flashinfer_trtllm as MoE runner backend on sm100 for DeepseekV3ForCausalLM | |
| [2026-06-26 19:58:56] DP attention is enabled. The chunked prefill size is adjusted to -1 to avoid MoE kernel issues. | |
| [2026-06-26 19:58:56] FlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set. | |
| [2026-06-26 19:58:59] server_args=ServerArgs(model_path='nvidia/GLM-5.2-NVFP4', tokenizer_path='zai-org/GLM-5.2', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=400000, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', host='127.0.0.1', port=30000, fastapi_root_path='', grpc_mode=False, skip_server_warmup=False, warmups=None, enable_http2=False, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, dtype='auto', quantization='modelopt_fp4', quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.8, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=-1, enable_dynamic_chunking=False, max_prefill_tokens=8192, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, schedule_conservativeness=0.3, page_size=64, swa_full_tokens_ratio=0.8, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=1023649404, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, nccl_port=None, dist_timeout=None, dist_init_addr=None, nnodes=1, node_rank=0, tp_size=4, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=4, load_balance_method='round_robin', attn_cp_size=1, moe_dp_size=1, enable_prefill_cp=False, cp_strategy=None, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_dp_attention=True, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=True, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, grpc_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, api_key=None, admin_api_key=None, served_model_name='glm52-nvfp4-dp4-upstream', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser=None, tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, attention_backend='dsa', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='pytorch', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='flashinfer_trtllm', dsa_prefill_backend='trtllm', dsa_decode_backend='trtllm', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, mamba_backend='triton', speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_window_size=None, speculative_moe_runner_backend='flashinfer_trtllm', speculative_moe_a2a_backend=None, speculative_draft_model_quantization='modelopt_fp4', speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=1, moe_a2a_backend='none', moe_runner_backend='flashinfer_trtllm', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, enable_expert_distribution_metrics=False, deepep_config=None, moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_deepep_waterfill=False, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=True, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_lmcache=False, lmcache_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=256, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256], tc_compiler='eager'), prefill=PhaseConfig(backend='disabled', max_bs=2048, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], tc_compiler='eager')), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=256, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=True, enable_mscclpp=False, enable_torch_symm_mem=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, torchao_config='', enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_retries=0, encoder_only=False, language_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, limit_mm_data_per_request=None, enable_mm_global_cache=False, disable_fast_image_processor=False, keep_mm_feature_on_device=False, forward_hooks=None, msprobe_dump_config=None) | |
| [2026-06-26 19:58:59] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:03] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:04] Using default HuggingFace chat template with detected content format: openai | |
| [2026-06-26 19:59:04] Auto-detected template features: reasoning_config=ReasoningToggleConfig(toggle_param='enable_thinking', default_enabled=True, special_case=None), reasoning_parser=glm45, tool_call_parser=glm47 | |
| [2026-06-26 19:59:20] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:33 DP3 TP3] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:33 DP2 TP2] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:34 DP0 TP0] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:34 DP1 TP1] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:37 DP3 TP3] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:37 DP2 TP2] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:38 DP1 TP1] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:38 DP0 TP0] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 19:59:38 DP3 TP3] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:38 DP2 TP2] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:38 DP3 TP3] Init torch distributed begin. | |
| [2026-06-26 19:59:38 DP2 TP2] Init torch distributed begin. | |
| [2026-06-26 19:59:38 DP1 TP1] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:38 DP0 TP0] DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell. | |
| [2026-06-26 19:59:38 DP1 TP1] Init torch distributed begin. | |
| [2026-06-26 19:59:38 DP0 TP0] Init torch distributed begin. | |
| [2026-06-26 19:59:39 DP0 TP0] sglang is using nccl==2.30.7 | |
| [2026-06-26 19:59:40 DP0 TP0] DCP disabled, dcp_size=1, tp_size=4 | |
| [2026-06-26 19:59:40 DP0 TP0] Init torch distributed ends. elapsed=1.88 s, mem usage=1.42 GB | |
| [2026-06-26 19:59:40 DP3 TP3] Init torch distributed ends. elapsed=2.27 s, mem usage=1.30 GB | |
| [2026-06-26 19:59:40 DP1 TP1] Init torch distributed ends. elapsed=1.91 s, mem usage=1.46 GB | |
| [2026-06-26 19:59:40 DP2 TP2] Init torch distributed ends. elapsed=2.27 s, mem usage=1.46 GB | |
| [2026-06-26 19:59:43 DP2 TP2] Ignore import error when loading sglang.srt.models.mimo_audio: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP3 TP3] Ignore import error when loading sglang.srt.models.mimo_audio: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP2 TP2] Ignore import error when loading sglang.srt.models.mimo_v2: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP3 TP3] Ignore import error when loading sglang.srt.models.mimo_v2: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP2 TP2] Ignore import error when loading sglang.srt.models.mimo_v2_asr: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP3 TP3] Ignore import error when loading sglang.srt.models.mimo_v2_asr: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP0 TP0] Ignore import error when loading sglang.srt.models.mimo_audio: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP1 TP1] Ignore import error when loading sglang.srt.models.mimo_audio: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP2 TP2] Ignore import error when loading sglang.srt.models.mimo_v2_nextn: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP3 TP3] Ignore import error when loading sglang.srt.models.mimo_v2_nextn: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP1 TP1] Ignore import error when loading sglang.srt.models.mimo_v2: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP0 TP0] Ignore import error when loading sglang.srt.models.mimo_v2: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP0 TP0] Ignore import error when loading sglang.srt.models.mimo_v2_asr: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP1 TP1] Ignore import error when loading sglang.srt.models.mimo_v2_asr: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP0 TP0] Ignore import error when loading sglang.srt.models.mimo_v2_nextn: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:43 DP1 TP1] Ignore import error when loading sglang.srt.models.mimo_v2_nextn: Can not import FA3 in sgl_kernel. Please check your installation. | |
| [2026-06-26 19:59:44 DP1 TP1] Load weight begin. avail mem=274.43 GB | |
| [2026-06-26 19:59:44 DP1 TP1] Using ModelOptModelLoader due to ModelOpt quantization config. | |
| [2026-06-26 19:59:44 DP1 TP1] ModelOptModelLoader: Loading base model... | |
| [2026-06-26 19:59:44 DP1 TP1] Model is already quantized, loading directly... | |
| [2026-06-26 19:59:44 DP1 TP1] Detected nvfp4 checkpoint. Please note that the format is experimental and subject to change. | |
| [2026-06-26 19:59:44 DP2 TP2] Load weight begin. avail mem=274.43 GB | |
| [2026-06-26 19:59:44 DP2 TP2] Using ModelOptModelLoader due to ModelOpt quantization config. | |
| [2026-06-26 19:59:44 DP2 TP2] ModelOptModelLoader: Loading base model... | |
| [2026-06-26 19:59:44 DP2 TP2] Model is already quantized, loading directly... | |
| [2026-06-26 19:59:44 DP2 TP2] Detected nvfp4 checkpoint. Please note that the format is experimental and subject to change. | |
| [2026-06-26 19:59:44 DP3 TP3] Load weight begin. avail mem=274.58 GB | |
| [2026-06-26 19:59:44 DP3 TP3] Using ModelOptModelLoader due to ModelOpt quantization config. | |
| [2026-06-26 19:59:44 DP3 TP3] ModelOptModelLoader: Loading base model... | |
| [2026-06-26 19:59:44 DP3 TP3] Model is already quantized, loading directly... | |
| [2026-06-26 19:59:44 DP3 TP3] Detected nvfp4 checkpoint. Please note that the format is experimental and subject to change. | |
| [2026-06-26 19:59:44 DP0 TP0] Load weight begin. avail mem=274.46 GB | |
| [2026-06-26 19:59:44 DP0 TP0] Using ModelOptModelLoader due to ModelOpt quantization config. | |
| [2026-06-26 19:59:44 DP0 TP0] ModelOptModelLoader: Loading base model... | |
| [2026-06-26 19:59:44 DP0 TP0] Model is already quantized, loading directly... | |
| [2026-06-26 19:59:44 DP0 TP0] Detected nvfp4 checkpoint. Please note that the format is experimental and subject to change. | |
| [2026-06-26 19:59:44 DP1 TP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_trtllm, quant_method=ModelOptNvFp4FusedMoEMethod). | |
| [2026-06-26 19:59:44 DP2 TP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_trtllm, quant_method=ModelOptNvFp4FusedMoEMethod). | |
| [2026-06-26 19:59:44 DP1 TP1] Setting inplace to False for FlashInfer TRTLLM MoE backend. | |
| [2026-06-26 19:59:44 DP3 TP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_trtllm, quant_method=ModelOptNvFp4FusedMoEMethod). | |
| [2026-06-26 19:59:44 DP2 TP2] Setting inplace to False for FlashInfer TRTLLM MoE backend. | |
| [2026-06-26 19:59:44 DP3 TP3] Setting inplace to False for FlashInfer TRTLLM MoE backend. | |
| [2026-06-26 19:59:44 DP0 TP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_trtllm, quant_method=ModelOptNvFp4FusedMoEMethod). | |
| [2026-06-26 19:59:44 DP0 TP0] Setting inplace to False for FlashInfer TRTLLM MoE backend. | |
| [2026-06-26 19:59:46 DP0 TP0] Found local HF snapshot for nvidia/GLM-5.2-NVFP4 at /tmp/sam/huggingface/hub/models--nvidia--GLM-5.2-NVFP4/snapshots/e1653eb5f7e2f965e6aedc410f2e9879515af5d7; skipping download. | |
| Multi-thread loading shards: 0% Completed | 0/47 [00:00<?, ?it/s] Multi-thread loading shards: 2% Completed | 1/47 [00:03<02:57, 3.87s/it] Multi-thread loading shards: 4% Completed | 2/47 [00:05<01:48, 2.41s/it] Multi-thread loading shards: 6% Completed | 3/47 [00:06<01:27, 1.98s/it] Multi-thread loading shards: 9% Completed | 4/47 [00:08<01:25, 1.98s/it] Multi-thread loading shards: 11% Completed | 5/47 [00:09<00:58, 1.39s/it] Multi-thread loading shards: 13% Completed | 6/47 [00:09<00:40, 1.01it/s] Multi-thread loading shards: 15% Completed | 7/47 [00:09<00:29, 1.36it/s] Multi-thread loading shards: 17% Completed | 8/47 [00:09<00:22, 1.75it/s] Multi-thread loading shards: 19% Completed | 9/47 [00:10<00:24, 1.54it/s] Multi-thread loading shards: 21% Completed | 10/47 [00:10<00:19, 1.94it/s] Multi-thread loading shards: 23% Completed | 11/47 [00:10<00:15, 2.35it/s] Multi-thread loading shards: 26% Completed | 12/47 [00:12<00:28, 1.23it/s] Multi-thread loading shards: 28% Completed | 13/47 [00:16<00:56, 1.67s/it] Multi-thread loading shards: 30% Completed | 14/47 [00:20<01:20, 2.45s/it] Multi-thread loading shards: 32% Completed | 15/47 [00:21<01:00, 1.90s/it] Multi-thread loading shards: 34% Completed | 16/47 [00:21<00:42, 1.38s/it] Multi-thread loading shards: 36% Completed | 17/47 [00:21<00:30, 1.03s/it] Multi-thread loading shards: 38% Completed | 18/47 [00:21<00:22, 1.27it/s] Multi-thread loading shards: 40% Completed | 19/47 [00:21<00:17, 1.64it/s] Multi-thread loading shards: 43% Completed | 20/47 [00:23<00:20, 1.31it/s] Multi-thread loading shards: 45% Completed | 21/47 [00:27<00:44, 1.71s/it] Multi-thread loading shards: 47% Completed | 22/47 [00:31<01:04, 2.57s/it] Multi-thread loading shards: 49% Completed | 23/47 [00:31<00:44, 1.86s/it] Multi-thread loading shards: 51% Completed | 24/47 [00:32<00:32, 1.41s/it] Multi-thread loading shards: 53% Completed | 25/47 [00:32<00:25, 1.15s/it] Multi-thread loading shards: 55% Completed | 26/47 [00:32<00:18, 1.13it/s] Multi-thread loading shards: 57% Completed | 27/47 [00:33<00:13, 1.47it/s] Multi-thread loading shards: 60% Completed | 28/47 [00:35<00:19, 1.04s/it] Multi-thread loading shards: 62% Completed | 29/47 [00:38<00:33, 1.85s/it] Multi-thread loading shards: 64% Completed | 30/47 [00:42<00:41, 2.43s/it] Multi-thread loading shards: 66% Completed | 31/47 [00:42<00:28, 1.80s/it] Multi-thread loading shards: 68% Completed | 32/47 [00:43<00:22, 1.48s/it] Multi-thread loading shards: 70% Completed | 33/47 [00:43<00:15, 1.09s/it] Multi-thread loading shards: 72% Completed | 34/47 [00:44<00:10, 1.20it/s] Multi-thread loading shards: 74% Completed | 35/47 [00:44<00:07, 1.54it/s] Multi-thread loading shards: 77% Completed | 36/47 [00:45<00:08, 1.26it/s] Multi-thread loading shards: 79% Completed | 37/47 [00:49<00:18, 1.80s/it] Multi-thread loading shards: 81% Completed | 38/47 [00:53<00:21, 2.39s/it] Multi-thread loading shards: 83% Completed | 39/47 [00:53<00:14, 1.81s/it] Multi-thread loading shards: 85% Completed | 40/47 [00:53<00:09, 1.33s/it] Multi-thread loading shards: 87% Completed | 41/47 [00:54<00:06, 1.02s/it] Multi-thread loading shards: 89% Completed | 42/47 [00:54<00:03, 1.28it/s] Multi-thread loading shards: 91% Completed | 43/47 [00:54<00:02, 1.61it/s] Multi-thread loading shards: 94% Completed | 44/47 [00:55<00:01, 1.64it/s] Multi-thread loading shards: 98% Completed | 46/47 [00:55<00:00, 2.70it/s] Multi-thread loading shards: 100% Completed | 47/47 [00:55<00:00, 1.18s/it] | |
| [2026-06-26 20:00:56 DP1 TP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! | |
| [2026-06-26 20:00:56 DP1 TP1] Load weight end. elapsed=72.86 s, type=GlmMoeDsaForCausalLM, quant=modelopt_fp4, quant_algo=NVFP4, avail mem=146.23 GB, mem usage=128.20 GB. | |
| [2026-06-26 20:00:58 DP0 TP0] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! | |
| [2026-06-26 20:00:58 DP0 TP0] Load weight end. elapsed=73.97 s, type=GlmMoeDsaForCausalLM, quant=modelopt_fp4, quant_algo=NVFP4, avail mem=146.26 GB, mem usage=128.20 GB. | |
| [2026-06-26 20:00:58 DP3 TP3] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! | |
| [2026-06-26 20:00:58 DP3 TP3] Load weight end. elapsed=74.28 s, type=GlmMoeDsaForCausalLM, quant=modelopt_fp4, quant_algo=NVFP4, avail mem=146.38 GB, mem usage=128.20 GB. | |
| [2026-06-26 20:00:58 DP2 TP2] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results! | |
| [2026-06-26 20:00:58 DP2 TP2] Load weight end. elapsed=74.84 s, type=GlmMoeDsaForCausalLM, quant=modelopt_fp4, quant_algo=NVFP4, avail mem=146.23 GB, mem usage=128.20 GB. | |
| [2026-06-26 20:01:03 DP3 TP3] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 20:01:03 DP2 TP2] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 20:01:03 DP1 TP1] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 20:01:04 DP0 TP0] Tokenizer for zai-org/GLM-5.2 is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing. | |
| [2026-06-26 20:01:04 DP3 TP3] KV Cache is allocated. dtype: torch.float8_e4m3fn, #tokens: 1775808, KV size: 91.34 GB | |
| [2026-06-26 20:01:04 DP2 TP2] KV Cache is allocated. dtype: torch.float8_e4m3fn, #tokens: 1775808, KV size: 91.34 GB | |
| [2026-06-26 20:01:04 DP3 TP3] Memory pool end. avail mem=54.88 GB | |
| [2026-06-26 20:01:04 DP2 TP2] Memory pool end. avail mem=54.72 GB | |
| [2026-06-26 20:01:04 DP0 TP0] KV Cache is allocated. dtype: torch.float8_e4m3fn, #tokens: 1775808, KV size: 91.34 GB | |
| [2026-06-26 20:01:04 DP0 TP0] Memory pool end. avail mem=54.75 GB | |
| [2026-06-26 20:01:04 DP1 TP1] KV Cache is allocated. dtype: torch.float8_e4m3fn, #tokens: 1775808, KV size: 91.34 GB | |
| [2026-06-26 20:01:04 DP1 TP1] Memory pool end. avail mem=54.72 GB | |
| [2026-06-26 20:01:04 DP3 TP3] Running FlashInfer autotune with cache: /home/sam/.cache/sglang/flashinfer/autotune/0.6.12/sm103/320505fa55d3aa1f/rank_tp3_pp0_dp3.json | |
| [2026-06-26 20:01:04 DP1 TP1] Running FlashInfer autotune with cache: /home/sam/.cache/sglang/flashinfer/autotune/0.6.12/sm103/320505fa55d3aa1f/rank_tp1_pp0_dp1.json | |
| [2026-06-26 20:01:04 DP0 TP0] Running FlashInfer autotune with cache: /home/sam/.cache/sglang/flashinfer/autotune/0.6.12/sm103/320505fa55d3aa1f/rank_tp0_pp0_dp0.json | |
| [2026-06-26 20:01:04 DP2 TP2] Running FlashInfer autotune with cache: /home/sam/.cache/sglang/flashinfer/autotune/0.6.12/sm103/320505fa55d3aa1f/rank_tp2_pp0_dp2.json | |
| 2026-06-26 20:01:05,196 - CUTE_DSL - WARNING - [handle_import_error] - Unexpected error during package walk: cutlass.cute.experimental | |
| [2026-06-26 20:01:05 DP1 TP1] Unexpected error during package walk: cutlass.cute.experimental | |
| 2026-06-26 20:01:05,199 - CUTE_DSL - WARNING - [handle_import_error] - Unexpected error during package walk: cutlass.cute.experimental | |
| [2026-06-26 20:01:05 DP2 TP2] Unexpected error during package walk: cutlass.cute.experimental | |
| 2026-06-26 20:01:05,200 - CUTE_DSL - WARNING - [handle_import_error] - Unexpected error during package walk: cutlass.cute.experimental | |
| [2026-06-26 20:01:05 DP3 TP3] Unexpected error during package walk: cutlass.cute.experimental | |
| 2026-06-26 20:01:05,203 - CUTE_DSL - WARNING - [handle_import_error] - Unexpected error during package walk: cutlass.cute.experimental | |
| [2026-06-26 20:01:05 DP0 TP0] Unexpected error during package walk: cutlass.cute.experimental | |
| [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 0%| | 0/9 [00:00<?, ?profile/s] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 0%| | 0/9 [00:00<?, ?profile/s] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 0%| | 0/9 [00:00<?, ?profile/s] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 0%| | 0/9 [00:00<?, ?profile/s] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 11%|█ | 1/9 [00:08<01:06, 8.31s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 11%|█ | 1/9 [00:08<01:05, 8.21s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 11%|█ | 1/9 [00:08<01:06, 8.31s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 11%|█ | 1/9 [00:08<01:06, 8.27s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 22%|██▏ | 2/9 [00:14<00:49, 7.02s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 22%|██▏ | 2/9 [00:14<00:49, 7.13s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 22%|██▏ | 2/9 [00:14<00:49, 7.14s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 22%|██▏ | 2/9 [00:14<00:50, 7.18s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 33%|███▎ | 3/9 [00:20<00:40, 6.77s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 33%|███▎ | 3/9 [00:21<00:41, 6.87s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 33%|███▎ | 3/9 [00:21<00:41, 6.90s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 33%|███▎ | 3/9 [00:21<00:41, 6.99s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 44%|████▍ | 4/9 [00:27<00:34, 6.82s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 44%|████▍ | 4/9 [00:28<00:34, 6.88s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 44%|████▍ | 4/9 [00:28<00:34, 6.94s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 44%|████▍ | 4/9 [00:28<00:35, 7.00s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 56%|█████▌ | 5/9 [00:35<00:28, 7.07s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 56%|█████▌ | 5/9 [00:35<00:28, 7.16s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 56%|█████▌ | 5/9 [00:35<00:28, 7.18s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 56%|█████▌ | 5/9 [00:36<00:29, 7.31s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 67%|██████▋ | 6/9 [00:43<00:22, 7.44s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 67%|██████▋ | 6/9 [00:43<00:22, 7.54s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 67%|██████▋ | 6/9 [00:44<00:22, 7.56s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 67%|██████▋ | 6/9 [00:44<00:23, 7.72s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 78%|███████▊ | 7/9 [00:52<00:15, 7.80s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 78%|███████▊ | 7/9 [00:52<00:15, 7.91s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 78%|███████▊ | 7/9 [00:52<00:15, 7.93s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 78%|███████▊ | 7/9 [00:53<00:16, 8.06s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 89%|████████▉ | 8/9 [01:00<00:08, 8.12s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 89%|████████▉ | 8/9 [01:01<00:08, 8.19s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 89%|████████▉ | 8/9 [01:01<00:08, 8.24s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 89%|████████▉ | 8/9 [01:02<00:08, 8.40s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:09<00:00, 8.38s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:09<00:00, 7.75s/profile] | |
| [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:10<00:00, 8.48s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:10<00:00, 7.84s/profile] | |
| [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:10<00:00, 8.49s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:10<00:00, 7.87s/profile] | |
| [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:11<00:00, 8.66s/profile] [AutoTuner]: Tuning flashinfer::trtllm_fp4_block_scale_moe: 100%|██████████| 9/9 [01:11<00:00, 8.00s/profile] | |
| [2026-06-26 20:02:27 DP3 TP3] FlashInfer autotune completed. | |
| [2026-06-26 20:02:27 DP3 TP3] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). | |
| [2026-06-26 20:02:27 DP2 TP2] FlashInfer autotune completed. | |
| [2026-06-26 20:02:27 DP2 TP2] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). | |
| [2026-06-26 20:02:27 DP1 TP1] FlashInfer autotune completed. | |
| [2026-06-26 20:02:27 DP1 TP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). | |
| [2026-06-26 20:02:27 DP2 TP2] Capture target decode CUDA graph begin. backend=full, num_tokens_per_bs=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=53.20 GB | |
| [2026-06-26 20:02:27 DP1 TP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_bs=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=53.20 GB | |
| [2026-06-26 20:02:27 DP3 TP3] Capture target decode CUDA graph begin. backend=full, num_tokens_per_bs=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=53.52 GB | |
| [2026-06-26 20:02:27 DP0 TP0] FlashInfer autotune completed. | |
| [2026-06-26 20:02:27 DP0 TP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules). | |
| [2026-06-26 20:02:27 DP0 TP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_bs=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=53.27 GB | |
| 0%| | 0/12 [00:00<?, ?it/s] Capturing batches (bs=64 avail_mem=53.13 GB): 0%| | 0/12 [00:00<?, ?it/s] Capturing batches (bs=64 avail_mem=53.13 GB): 8%|▊ | 1/12 [00:02<00:25, 2.34s/it] Capturing batches (bs=56 avail_mem=52.80 GB): 8%|▊ | 1/12 [00:02<00:25, 2.34s/it] Capturing batches (bs=56 avail_mem=52.80 GB): 17%|█▋ | 2/12 [00:03<00:13, 1.36s/it] Capturing batches (bs=48 avail_mem=52.75 GB): 17%|█▋ | 2/12 [00:03<00:13, 1.36s/it] Capturing batches (bs=48 avail_mem=52.75 GB): 25%|██▌ | 3/12 [00:03<00:09, 1.05s/it] Capturing batches (bs=40 avail_mem=52.69 GB): 25%|██▌ | 3/12 [00:03<00:09, 1.05s/it] Capturing batches (bs=40 avail_mem=52.69 GB): 33%|███▎ | 4/12 [00:04<00:07, 1.10it/s] Capturing batches (bs=32 avail_mem=52.64 GB): 33%|███▎ | 4/12 [00:04<00:07, 1.10it/s] Capturing batches (bs=32 avail_mem=52.64 GB): 42%|████▏ | 5/12 [00:06<00:08, 1.23s/it] Capturing batches (bs=24 avail_mem=52.58 GB): 42%|████▏ | 5/12 [00:06<00:08, 1.23s/it] Capturing batches (bs=24 avail_mem=52.58 GB): 50%|█████ | 6/12 [00:06<00:06, 1.06s/it] Capturing batches (bs=16 avail_mem=52.52 GB): 50%|█████ | 6/12 [00:06<00:06, 1.06s/it] Capturing batches (bs=16 avail_mem=52.52 GB): 58%|█████▊ | 7/12 [00:07<00:04, 1.06it/s] Capturing batches (bs=12 avail_mem=52.47 GB): 58%|█████▊ | 7/12 [00:07<00:04, 1.06it/s] Capturing batches (bs=12 avail_mem=52.47 GB): 67%|██████▋ | 8/12 [00:08<00:03, 1.15it/s] Capturing batches (bs=8 avail_mem=52.42 GB): 67%|██████▋ | 8/12 [00:08<00:03, 1.15it/s] Capturing batches (bs=8 avail_mem=52.42 GB): 75%|███████▌ | 9/12 [00:09<00:02, 1.19it/s] Capturing batches (bs=4 avail_mem=52.36 GB): 75%|███████▌ | 9/12 [00:09<00:02, 1.19it/s] Capturing batches (bs=4 avail_mem=52.36 GB): 83%|████████▎ | 10/12 [00:17<00:06, 3.25s/it] Capturing batches (bs=2 avail_mem=52.30 GB): 83%|████████▎ | 10/12 [00:17<00:06, 3.25s/it] Capturing batches (bs=2 avail_mem=52.30 GB): 92%|█████████▏| 11/12 [00:18<00:02, 2.45s/it] Capturing batches (bs=1 avail_mem=52.25 GB): 92%|█████████▏| 11/12 [00:18<00:02, 2.45s/it] Capturing batches (bs=1 avail_mem=52.25 GB): 100%|██████████| 12/12 [00:20<00:00, 2.49s/it] Capturing batches (bs=1 avail_mem=52.25 GB): 100%|██████████| 12/12 [00:20<00:00, 1.75s/it] | |
| [2026-06-26 20:02:49 DP2 TP2] Capture target decode CUDA graph end. elapsed=22.12 s, mem usage=1.07 GB, avail mem=52.13 GB. | |
| [2026-06-26 20:02:49 DP2 TP2] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False | |
| [2026-06-26 20:02:49 DP3 TP3] Capture target decode CUDA graph end. elapsed=22.13 s, mem usage=1.07 GB, avail mem=52.45 GB. | |
| [2026-06-26 20:02:49 DP3 TP3] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False | |
| [2026-06-26 20:02:49 DP0 TP0] Capture target decode CUDA graph end. elapsed=21.98 s, mem usage=1.07 GB, avail mem=52.20 GB. | |
| [2026-06-26 20:02:49 DP1 TP1] Capture target decode CUDA graph end. elapsed=22.14 s, mem usage=1.07 GB, avail mem=52.13 GB. | |
| [2026-06-26 20:02:49 DP0 TP0] max_total_num_tokens=1775808, chunked_prefill_size=-1, max_prefill_tokens=8192, max_running_requests=64, context_len=400000, available_gpu_mem=52.20 GB | |
| [2026-06-26 20:02:49 DP0 TP0] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False | |
| [2026-06-26 20:02:49 DP1 TP1] Tree cache initialized: source=default impl=RadixCache hybrid_swa=False hybrid_ssm=False hierarchical=False streaming_wrapped=False | |
| [2026-06-26 20:02:50] INFO: Started server process [63407] | |
| [2026-06-26 20:02:50] INFO: Waiting for application startup. | |
| [2026-06-26 20:02:50] Using default chat sampling params from model generation config: {'temperature': 1.0, 'top_p': 0.95} | |
| [2026-06-26 20:02:50] INFO: Application startup complete. | |
| [2026-06-26 20:02:50] INFO: Uvicorn running on http://127.0.0.1:30000 (Press CTRL+C to quit) | |
| [2026-06-26 20:02:51] INFO: 127.0.0.1:32806 - "GET /model_info HTTP/1.1" 200 OK | |
| [2026-06-26 20:02:56 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.96 | |
| [2026-06-26 20:02:56 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.86 | |
| [2026-06-26 20:02:56 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.73 | |
| [2026-06-26 20:02:56 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 8.63 | |
| [2026-06-26 20:02:57] INFO: 127.0.0.1:32822 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:02:57] The server is fired up and ready to roll! | |
| [2026-06-26 20:03:25 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 2.22 | |
| [2026-06-26 20:03:26] INFO: 127.0.0.1:45436 - "GET /health HTTP/1.1" 200 OK | |
| [2026-06-26 20:03:53 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.03, #running-req: 0, #queue-req: 30, #pending-token: 0, cuda graph: False, input throughput (token/s): 407.33 | |
| [2026-06-26 20:03:55 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.04, #running-req: 1, #queue-req: 29, #pending-token: 688638, cuda graph: False, input throughput (token/s): 12085.02 | |
| [2026-06-26 20:03:55 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.03, #running-req: 0, #queue-req: 30, #pending-token: 711594, cuda graph: False, input throughput (token/s): 772.50 | |
| [2026-06-26 20:03:55 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.03, #running-req: 0, #queue-req: 30, #pending-token: 711591, cuda graph: False, input throughput (token/s): 393.35 | |
| [2026-06-26 20:03:55 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.03, #running-req: 0, #queue-req: 30, #pending-token: 711592, cuda graph: False, input throughput (token/s): 392.69 | |
| [2026-06-26 20:03:57 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.04, #running-req: 1, #queue-req: 29, #pending-token: 688638, cuda graph: False, input throughput (token/s): 12294.06 | |
| [2026-06-26 20:03:57 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.04, #running-req: 1, #queue-req: 29, #pending-token: 688638, cuda graph: False, input throughput (token/s): 12294.05 | |
| [2026-06-26 20:03:57 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.04, #running-req: 1, #queue-req: 29, #pending-token: 688638, cuda graph: False, input throughput (token/s): 12293.26 | |
| [2026-06-26 20:03:57 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.05, #running-req: 2, #queue-req: 28, #pending-token: 665684, cuda graph: False, input throughput (token/s): 12285.58 | |
| [2026-06-26 20:03:58 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.06, #running-req: 3, #queue-req: 27, #pending-token: 642728, cuda graph: False, input throughput (token/s): 12300.45 | |
| [2026-06-26 20:03:59 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.05, #running-req: 2, #queue-req: 28, #pending-token: 665682, cuda graph: False, input throughput (token/s): 12287.54 | |
| [2026-06-26 20:03:59 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.05, #running-req: 2, #queue-req: 28, #pending-token: 665685, cuda graph: False, input throughput (token/s): 12287.80 | |
| [2026-06-26 20:03:59 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.05, #running-req: 2, #queue-req: 28, #pending-token: 665684, cuda graph: False, input throughput (token/s): 12287.36 | |
| [2026-06-26 20:04:00 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.06, #running-req: 3, #queue-req: 27, #pending-token: 642729, cuda graph: False, input throughput (token/s): 12295.07 | |
| [2026-06-26 20:04:00 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.06, #running-req: 3, #queue-req: 27, #pending-token: 642728, cuda graph: False, input throughput (token/s): 12294.84 | |
| [2026-06-26 20:04:00 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.08, #running-req: 4, #queue-req: 26, #pending-token: 619774, cuda graph: False, input throughput (token/s): 12289.38 | |
| [2026-06-26 20:04:00 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.06, #running-req: 3, #queue-req: 27, #pending-token: 642731, cuda graph: False, input throughput (token/s): 12295.13 | |
| [2026-06-26 20:04:02 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.08, #running-req: 4, #queue-req: 26, #pending-token: 619772, cuda graph: False, input throughput (token/s): 12367.26 | |
| [2026-06-26 20:04:02 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.08, #running-req: 4, #queue-req: 26, #pending-token: 619775, cuda graph: False, input throughput (token/s): 12366.82 | |
| [2026-06-26 20:04:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.09, #running-req: 5, #queue-req: 25, #pending-token: 596821, cuda graph: False, input throughput (token/s): 12367.11 | |
| [2026-06-26 20:04:02 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.08, #running-req: 4, #queue-req: 26, #pending-token: 619775, cuda graph: False, input throughput (token/s): 12366.79 | |
| [2026-06-26 20:04:04 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.10, #running-req: 6, #queue-req: 24, #pending-token: 573865, cuda graph: False, input throughput (token/s): 12196.29 | |
| [2026-06-26 20:04:04 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.09, #running-req: 5, #queue-req: 25, #pending-token: 596821, cuda graph: False, input throughput (token/s): 12196.77 | |
| [2026-06-26 20:04:04 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.09, #running-req: 5, #queue-req: 25, #pending-token: 596819, cuda graph: False, input throughput (token/s): 12195.05 | |
| [2026-06-26 20:04:04 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.09, #running-req: 5, #queue-req: 25, #pending-token: 596818, cuda graph: False, input throughput (token/s): 12194.81 | |
| [2026-06-26 20:04:06 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.10, #running-req: 6, #queue-req: 24, #pending-token: 573865, cuda graph: False, input throughput (token/s): 12289.87 | |
| [2026-06-26 20:04:06 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.10, #running-req: 6, #queue-req: 24, #pending-token: 573865, cuda graph: False, input throughput (token/s): 12289.72 | |
| [2026-06-26 20:04:06 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.12, #running-req: 7, #queue-req: 23, #pending-token: 550911, cuda graph: False, input throughput (token/s): 12287.88 | |
| [2026-06-26 20:04:06 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.10, #running-req: 6, #queue-req: 24, #pending-token: 573865, cuda graph: False, input throughput (token/s): 12287.89 | |
| [2026-06-26 20:04:08 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.12, #running-req: 7, #queue-req: 23, #pending-token: 550912, cuda graph: False, input throughput (token/s): 12285.07 | |
| [2026-06-26 20:04:08 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.12, #running-req: 7, #queue-req: 23, #pending-token: 550909, cuda graph: False, input throughput (token/s): 12284.83 | |
| [2026-06-26 20:04:08 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.12, #running-req: 7, #queue-req: 23, #pending-token: 550911, cuda graph: False, input throughput (token/s): 12285.21 | |
| [2026-06-26 20:04:08 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.13, #running-req: 8, #queue-req: 22, #pending-token: 527955, cuda graph: False, input throughput (token/s): 12281.87 | |
| [2026-06-26 20:04:10 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.14, #running-req: 9, #queue-req: 21, #pending-token: 505001, cuda graph: False, input throughput (token/s): 12289.78 | |
| [2026-06-26 20:04:10 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.13, #running-req: 8, #queue-req: 22, #pending-token: 527956, cuda graph: False, input throughput (token/s): 12280.01 | |
| [2026-06-26 20:04:10 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.13, #running-req: 8, #queue-req: 22, #pending-token: 527955, cuda graph: False, input throughput (token/s): 12279.96 | |
| [2026-06-26 20:04:10 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.13, #running-req: 8, #queue-req: 22, #pending-token: 527958, cuda graph: False, input throughput (token/s): 12279.50 | |
| [2026-06-26 20:04:12 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.16, #running-req: 10, #queue-req: 20, #pending-token: 482048, cuda graph: False, input throughput (token/s): 12382.15 | |
| [2026-06-26 20:04:12 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.14, #running-req: 9, #queue-req: 21, #pending-token: 504999, cuda graph: False, input throughput (token/s): 12387.93 | |
| [2026-06-26 20:04:12 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.14, #running-req: 9, #queue-req: 21, #pending-token: 505002, cuda graph: False, input throughput (token/s): 12387.26 | |
| [2026-06-26 20:04:12 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.14, #running-req: 9, #queue-req: 21, #pending-token: 505002, cuda graph: False, input throughput (token/s): 12387.98 | |
| [2026-06-26 20:04:13 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.16, #running-req: 10, #queue-req: 20, #pending-token: 482045, cuda graph: False, input throughput (token/s): 12192.04 | |
| [2026-06-26 20:04:13 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.16, #running-req: 10, #queue-req: 20, #pending-token: 482046, cuda graph: False, input throughput (token/s): 12192.34 | |
| [2026-06-26 20:04:13 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.17, #running-req: 11, #queue-req: 19, #pending-token: 459092, cuda graph: False, input throughput (token/s): 12191.62 | |
| [2026-06-26 20:04:13 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.16, #running-req: 10, #queue-req: 20, #pending-token: 482048, cuda graph: False, input throughput (token/s): 12192.30 | |
| [2026-06-26 20:04:15 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.17, #running-req: 11, #queue-req: 19, #pending-token: 459092, cuda graph: False, input throughput (token/s): 12283.57 | |
| [2026-06-26 20:04:15 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.17, #running-req: 11, #queue-req: 19, #pending-token: 459092, cuda graph: False, input throughput (token/s): 12283.06 | |
| [2026-06-26 20:04:15 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.18, #running-req: 12, #queue-req: 18, #pending-token: 436138, cuda graph: False, input throughput (token/s): 12282.76 | |
| [2026-06-26 20:04:15 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.17, #running-req: 11, #queue-req: 19, #pending-token: 459092, cuda graph: False, input throughput (token/s): 12282.90 | |
| [2026-06-26 20:04:17 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.19, #running-req: 13, #queue-req: 17, #pending-token: 413182, cuda graph: False, input throughput (token/s): 12282.22 | |
| [2026-06-26 20:04:17 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.18, #running-req: 12, #queue-req: 18, #pending-token: 436138, cuda graph: False, input throughput (token/s): 12282.55 | |
| [2026-06-26 20:04:17 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.18, #running-req: 12, #queue-req: 18, #pending-token: 436139, cuda graph: False, input throughput (token/s): 12281.32 | |
| [2026-06-26 20:04:17 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.18, #running-req: 12, #queue-req: 18, #pending-token: 436136, cuda graph: False, input throughput (token/s): 12280.66 | |
| [2026-06-26 20:04:19 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.19, #running-req: 13, #queue-req: 17, #pending-token: 413182, cuda graph: False, input throughput (token/s): 12279.74 | |
| [2026-06-26 20:04:19 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.19, #running-req: 13, #queue-req: 17, #pending-token: 413183, cuda graph: False, input throughput (token/s): 12279.58 | |
| [2026-06-26 20:04:19 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.21, #running-req: 14, #queue-req: 16, #pending-token: 390228, cuda graph: False, input throughput (token/s): 12278.69 | |
| [2026-06-26 20:04:19 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.19, #running-req: 13, #queue-req: 17, #pending-token: 413185, cuda graph: False, input throughput (token/s): 12275.74 | |
| [2026-06-26 20:04:21 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.21, #running-req: 14, #queue-req: 16, #pending-token: 390226, cuda graph: False, input throughput (token/s): 12377.05 | |
| [2026-06-26 20:04:21 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.21, #running-req: 14, #queue-req: 16, #pending-token: 390229, cuda graph: False, input throughput (token/s): 12376.80 | |
| [2026-06-26 20:04:21 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.22, #running-req: 15, #queue-req: 15, #pending-token: 367275, cuda graph: False, input throughput (token/s): 12376.67 | |
| [2026-06-26 20:04:21 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.21, #running-req: 14, #queue-req: 16, #pending-token: 390229, cuda graph: False, input throughput (token/s): 12379.16 | |
| [2026-06-26 20:04:23 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.22, #running-req: 15, #queue-req: 15, #pending-token: 367272, cuda graph: False, input throughput (token/s): 12186.14 | |
| [2026-06-26 20:04:23 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.22, #running-req: 15, #queue-req: 15, #pending-token: 367273, cuda graph: False, input throughput (token/s): 12186.21 | |
| [2026-06-26 20:04:23 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.23, #running-req: 16, #queue-req: 14, #pending-token: 344319, cuda graph: False, input throughput (token/s): 12186.15 | |
| [2026-06-26 20:04:23 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.22, #running-req: 15, #queue-req: 15, #pending-token: 367275, cuda graph: False, input throughput (token/s): 12186.59 | |
| [2026-06-26 20:04:25 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.23, #running-req: 16, #queue-req: 14, #pending-token: 344319, cuda graph: False, input throughput (token/s): 12273.53 | |
| [2026-06-26 20:04:25 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.23, #running-req: 16, #queue-req: 14, #pending-token: 344319, cuda graph: False, input throughput (token/s): 12273.68 | |
| [2026-06-26 20:04:25 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.23, #running-req: 16, #queue-req: 14, #pending-token: 344319, cuda graph: False, input throughput (token/s): 12249.51 | |
| [2026-06-26 20:04:25 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.25, #running-req: 17, #queue-req: 13, #pending-token: 321365, cuda graph: False, input throughput (token/s): 12209.56 | |
| [2026-06-26 20:04:27 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.26, #running-req: 18, #queue-req: 12, #pending-token: 298409, cuda graph: False, input throughput (token/s): 12339.67 | |
| [2026-06-26 20:04:27 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.25, #running-req: 17, #queue-req: 13, #pending-token: 321365, cuda graph: False, input throughput (token/s): 12299.17 | |
| [2026-06-26 20:04:27 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.25, #running-req: 17, #queue-req: 13, #pending-token: 321366, cuda graph: False, input throughput (token/s): 12273.86 | |
| [2026-06-26 20:04:27 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.25, #running-req: 17, #queue-req: 13, #pending-token: 321363, cuda graph: False, input throughput (token/s): 12273.65 | |
| [2026-06-26 20:04:28 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.26, #running-req: 18, #queue-req: 12, #pending-token: 298412, cuda graph: False, input throughput (token/s): 12284.45 | |
| [2026-06-26 20:04:28 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.26, #running-req: 18, #queue-req: 12, #pending-token: 298409, cuda graph: False, input throughput (token/s): 12284.59 | |
| [2026-06-26 20:04:28 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.26, #running-req: 18, #queue-req: 12, #pending-token: 298410, cuda graph: False, input throughput (token/s): 12284.07 | |
| [2026-06-26 20:04:28 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.27, #running-req: 19, #queue-req: 11, #pending-token: 275455, cuda graph: False, input throughput (token/s): 12280.94 | |
| [2026-06-26 20:04:30 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.28, #running-req: 20, #queue-req: 10, #pending-token: 252502, cuda graph: False, input throughput (token/s): 12379.47 | |
| [2026-06-26 20:04:30 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.27, #running-req: 19, #queue-req: 11, #pending-token: 275456, cuda graph: False, input throughput (token/s): 12375.03 | |
| [2026-06-26 20:04:30 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.27, #running-req: 19, #queue-req: 11, #pending-token: 275453, cuda graph: False, input throughput (token/s): 12371.52 | |
| [2026-06-26 20:04:30 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.27, #running-req: 19, #queue-req: 11, #pending-token: 275456, cuda graph: False, input throughput (token/s): 12371.81 | |
| [2026-06-26 20:04:32 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.28, #running-req: 20, #queue-req: 10, #pending-token: 252500, cuda graph: False, input throughput (token/s): 12190.44 | |
| [2026-06-26 20:04:32 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.28, #running-req: 20, #queue-req: 10, #pending-token: 252499, cuda graph: False, input throughput (token/s): 12190.18 | |
| [2026-06-26 20:04:32 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.30, #running-req: 21, #queue-req: 9, #pending-token: 229546, cuda graph: False, input throughput (token/s): 12185.61 | |
| [2026-06-26 20:04:32 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.28, #running-req: 20, #queue-req: 10, #pending-token: 252502, cuda graph: False, input throughput (token/s): 12186.47 | |
| [2026-06-26 20:04:34 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.30, #running-req: 21, #queue-req: 9, #pending-token: 229546, cuda graph: False, input throughput (token/s): 12278.94 | |
| [2026-06-26 20:04:34 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.30, #running-req: 21, #queue-req: 9, #pending-token: 229546, cuda graph: False, input throughput (token/s): 12278.34 | |
| [2026-06-26 20:04:34 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.31, #running-req: 22, #queue-req: 8, #pending-token: 206592, cuda graph: False, input throughput (token/s): 12278.46 | |
| [2026-06-26 20:04:34 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.30, #running-req: 21, #queue-req: 9, #pending-token: 229546, cuda graph: False, input throughput (token/s): 12278.40 | |
| [2026-06-26 20:04:36 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.32, #running-req: 23, #queue-req: 7, #pending-token: 183636, cuda graph: False, input throughput (token/s): 12283.21 | |
| [2026-06-26 20:04:36 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.31, #running-req: 22, #queue-req: 8, #pending-token: 206592, cuda graph: False, input throughput (token/s): 12283.22 | |
| [2026-06-26 20:04:36 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.31, #running-req: 22, #queue-req: 8, #pending-token: 206590, cuda graph: False, input throughput (token/s): 12282.26 | |
| [2026-06-26 20:04:36 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.31, #running-req: 22, #queue-req: 8, #pending-token: 206593, cuda graph: False, input throughput (token/s): 12282.64 | |
| [2026-06-26 20:04:38 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.32, #running-req: 23, #queue-req: 7, #pending-token: 183637, cuda graph: False, input throughput (token/s): 12282.82 | |
| [2026-06-26 20:04:38 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.32, #running-req: 23, #queue-req: 7, #pending-token: 183636, cuda graph: False, input throughput (token/s): 12282.73 | |
| [2026-06-26 20:04:38 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.32, #running-req: 23, #queue-req: 7, #pending-token: 183639, cuda graph: False, input throughput (token/s): 12281.99 | |
| [2026-06-26 20:04:38 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.34, #running-req: 24, #queue-req: 6, #pending-token: 160682, cuda graph: False, input throughput (token/s): 12281.84 | |
| [2026-06-26 20:04:40 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.35, #running-req: 25, #queue-req: 5, #pending-token: 137729, cuda graph: False, input throughput (token/s): 12375.47 | |
| [2026-06-26 20:04:40 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.34, #running-req: 24, #queue-req: 6, #pending-token: 160680, cuda graph: False, input throughput (token/s): 12374.54 | |
| [2026-06-26 20:04:40 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.34, #running-req: 24, #queue-req: 6, #pending-token: 160683, cuda graph: False, input throughput (token/s): 12374.33 | |
| [2026-06-26 20:04:40 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.34, #running-req: 24, #queue-req: 6, #pending-token: 160683, cuda graph: False, input throughput (token/s): 12370.19 | |
| [2026-06-26 20:04:42 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.35, #running-req: 25, #queue-req: 5, #pending-token: 137729, cuda graph: False, input throughput (token/s): 12188.08 | |
| [2026-06-26 20:04:42 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.35, #running-req: 25, #queue-req: 5, #pending-token: 137727, cuda graph: False, input throughput (token/s): 12183.44 | |
| [2026-06-26 20:04:42 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.36, #running-req: 26, #queue-req: 4, #pending-token: 114773, cuda graph: False, input throughput (token/s): 12182.68 | |
| [2026-06-26 20:04:42 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.35, #running-req: 25, #queue-req: 5, #pending-token: 137726, cuda graph: False, input throughput (token/s): 12183.08 | |
| [2026-06-26 20:04:43 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.36, #running-req: 26, #queue-req: 4, #pending-token: 114773, cuda graph: False, input throughput (token/s): 12280.32 | |
| [2026-06-26 20:04:43 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.36, #running-req: 26, #queue-req: 4, #pending-token: 114773, cuda graph: False, input throughput (token/s): 12280.64 | |
| [2026-06-26 20:04:43 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.38, #running-req: 27, #queue-req: 3, #pending-token: 91819, cuda graph: False, input throughput (token/s): 12280.41 | |
| [2026-06-26 20:04:43 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.36, #running-req: 26, #queue-req: 4, #pending-token: 114773, cuda graph: False, input throughput (token/s): 12280.04 | |
| [2026-06-26 20:04:45 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.38, #running-req: 27, #queue-req: 3, #pending-token: 91817, cuda graph: False, input throughput (token/s): 12282.18 | |
| [2026-06-26 20:04:45 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.38, #running-req: 27, #queue-req: 3, #pending-token: 91820, cuda graph: False, input throughput (token/s): 12281.81 | |
| [2026-06-26 20:04:45 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.39, #running-req: 28, #queue-req: 2, #pending-token: 68863, cuda graph: False, input throughput (token/s): 12280.82 | |
| [2026-06-26 20:04:45 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.38, #running-req: 27, #queue-req: 3, #pending-token: 91819, cuda graph: False, input throughput (token/s): 12280.91 | |
| [2026-06-26 20:04:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.39, #running-req: 28, #queue-req: 2, #pending-token: 68864, cuda graph: False, input throughput (token/s): 12280.92 | |
| [2026-06-26 20:04:47 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.39, #running-req: 28, #queue-req: 2, #pending-token: 68863, cuda graph: False, input throughput (token/s): 12280.40 | |
| [2026-06-26 20:04:47 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.40, #running-req: 29, #queue-req: 1, #pending-token: 45909, cuda graph: False, input throughput (token/s): 12281.71 | |
| [2026-06-26 20:04:47 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.39, #running-req: 28, #queue-req: 2, #pending-token: 68866, cuda graph: False, input throughput (token/s): 12281.62 | |
| [2026-06-26 20:04:49 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.40, #running-req: 29, #queue-req: 1, #pending-token: 45907, cuda graph: False, input throughput (token/s): 12376.59 | |
| [2026-06-26 20:04:49 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.40, #running-req: 29, #queue-req: 1, #pending-token: 45910, cuda graph: False, input throughput (token/s): 12376.10 | |
| [2026-06-26 20:04:49 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.40, #running-req: 29, #queue-req: 1, #pending-token: 45910, cuda graph: False, input throughput (token/s): 12375.82 | |
| [2026-06-26 20:04:49 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 30, #queue-req: 0, #pending-token: 22956, cuda graph: False, input throughput (token/s): 12372.63 | |
| [2026-06-26 20:04:51 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 30, #queue-req: 0, #pending-token: 22956, cuda graph: False, input throughput (token/s): 13375.22 | |
| [2026-06-26 20:04:51 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 30, #queue-req: 0, #pending-token: 22954, cuda graph: False, input throughput (token/s): 13371.63 | |
| [2026-06-26 20:04:51 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 30, #queue-req: 0, #pending-token: 22953, cuda graph: False, input throughput (token/s): 13371.15 | |
| [2026-06-26 20:04:51 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 10072.36 | |
| [2026-06-26 20:04:51 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 38696.97 | |
| [2026-06-26 20:04:51 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 38695.93 | |
| [2026-06-26 20:04:51 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 22976, #cached-token: 0, token usage: 0.41, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 38628.48 | |
| [2026-06-26 20:04:52 DP1 TP1] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 8.39, #queue-req: 0 | |
| [2026-06-26 20:04:52 DP0 TP0] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 8.13, #queue-req: 0 | |
| [2026-06-26 20:04:52 DP2 TP2] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 8.38, #queue-req: 0 | |
| [2026-06-26 20:04:52 DP3 TP3] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 8.38, #queue-req: 0 | |
| [2026-06-26 20:04:53 DP0 TP0] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1156.33, #queue-req: 0 | |
| [2026-06-26 20:04:53 DP1 TP1] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1156.33, #queue-req: 0 | |
| [2026-06-26 20:04:53 DP2 TP2] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1155.88, #queue-req: 0 | |
| [2026-06-26 20:04:53 DP3 TP3] Decode batch, #running-req: 32, #token: 737280, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1155.90, #queue-req: 0 | |
| [2026-06-26 20:04:54 DP1 TP1] Decode batch, #running-req: 32, #token: 739328, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1149.26, #queue-req: 0 | |
| [2026-06-26 20:04:54 DP0 TP0] Decode batch, #running-req: 32, #token: 739328, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1149.28, #queue-req: 0 | |
| [2026-06-26 20:04:54 DP2 TP2] Decode batch, #running-req: 32, #token: 739328, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1149.28, #queue-req: 0 | |
| [2026-06-26 20:04:54 DP3 TP3] Decode batch, #running-req: 32, #token: 739328, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1149.27, #queue-req: 0 | |
| [2026-06-26 20:04:55 DP0 TP0] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1146.13, #queue-req: 0 | |
| [2026-06-26 20:04:55 DP1 TP1] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1146.14, #queue-req: 0 | |
| [2026-06-26 20:04:55 DP2 TP2] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.85, #queue-req: 0 | |
| [2026-06-26 20:04:55 DP3 TP3] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.86, #queue-req: 0 | |
| [2026-06-26 20:04:57 DP0 TP0] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.81, #queue-req: 0 | |
| [2026-06-26 20:04:57 DP1 TP1] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.81, #queue-req: 0 | |
| [2026-06-26 20:04:57 DP2 TP2] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.82, #queue-req: 0 | |
| [2026-06-26 20:04:57 DP3 TP3] Decode batch, #running-req: 32, #token: 741376, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.80, #queue-req: 0 | |
| [2026-06-26 20:04:58 DP0 TP0] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.58, #queue-req: 0 | |
| [2026-06-26 20:04:58 DP1 TP1] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.58, #queue-req: 0 | |
| [2026-06-26 20:04:58 DP2 TP2] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.73, #queue-req: 0 | |
| [2026-06-26 20:04:58 DP3 TP3] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.72, #queue-req: 0 | |
| [2026-06-26 20:04:59 DP0 TP0] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.93, #queue-req: 0 | |
| [2026-06-26 20:04:59 DP1 TP1] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.92, #queue-req: 0 | |
| [2026-06-26 20:04:59 DP2 TP2] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.86, #queue-req: 0 | |
| [2026-06-26 20:04:59 DP3 TP3] Decode batch, #running-req: 32, #token: 743424, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1143.86, #queue-req: 0 | |
| [2026-06-26 20:05:00 DP1 TP1] Decode batch, #running-req: 32, #token: 745472, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.57, #queue-req: 0 | |
| [2026-06-26 20:05:00 DP0 TP0] Decode batch, #running-req: 32, #token: 745472, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.57, #queue-req: 0 | |
| [2026-06-26 20:05:00 DP3 TP3] Decode batch, #running-req: 32, #token: 745472, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.82, #queue-req: 0 | |
| [2026-06-26 20:05:00 DP2 TP2] Decode batch, #running-req: 32, #token: 745472, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.81, #queue-req: 0 | |
| [2026-06-26 20:05:01 DP1 TP1] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.22, #queue-req: 0 | |
| [2026-06-26 20:05:01 DP0 TP0] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1145.21, #queue-req: 0 | |
| [2026-06-26 20:05:01 DP2 TP2] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1144.80, #queue-req: 0 | |
| [2026-06-26 20:05:01 DP3 TP3] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1144.80, #queue-req: 0 | |
| [2026-06-26 20:05:02 DP0 TP0] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.90, #queue-req: 0 | |
| [2026-06-26 20:05:02 DP1 TP1] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1141.90, #queue-req: 0 | |
| [2026-06-26 20:05:02 DP3 TP3] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1142.11, #queue-req: 0 | |
| [2026-06-26 20:05:02 DP2 TP2] Decode batch, #running-req: 32, #token: 747520, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1142.11, #queue-req: 0 | |
| [2026-06-26 20:05:03 DP1 TP1] Decode batch, #running-req: 32, #token: 749568, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.60, #queue-req: 0 | |
| [2026-06-26 20:05:03 DP0 TP0] Decode batch, #running-req: 32, #token: 749568, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.60, #queue-req: 0 | |
| [2026-06-26 20:05:03 DP2 TP2] Decode batch, #running-req: 32, #token: 749568, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.41, #queue-req: 0 | |
| [2026-06-26 20:05:03 DP3 TP3] Decode batch, #running-req: 32, #token: 749568, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.40, #queue-req: 0 | |
| [2026-06-26 20:05:04 DP1 TP1] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.33, #queue-req: 0 | |
| [2026-06-26 20:05:04 DP0 TP0] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.34, #queue-req: 0 | |
| [2026-06-26 20:05:04 DP3 TP3] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.44, #queue-req: 0 | |
| [2026-06-26 20:05:04 DP2 TP2] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1138.43, #queue-req: 0 | |
| [2026-06-26 20:05:06 DP1 TP1] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1137.09, #queue-req: 0 | |
| [2026-06-26 20:05:06 DP0 TP0] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1137.09, #queue-req: 0 | |
| [2026-06-26 20:05:06 DP3 TP3] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1136.89, #queue-req: 0 | |
| [2026-06-26 20:05:06 DP2 TP2] Decode batch, #running-req: 32, #token: 751616, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1136.88, #queue-req: 0 | |
| [2026-06-26 20:05:07 DP1 TP1] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1135.25, #queue-req: 0 | |
| [2026-06-26 20:05:07 DP0 TP0] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1135.26, #queue-req: 0 | |
| [2026-06-26 20:05:07 DP3 TP3] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1135.13, #queue-req: 0 | |
| [2026-06-26 20:05:07 DP2 TP2] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1135.14, #queue-req: 0 | |
| [2026-06-26 20:05:08 DP0 TP0] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1133.67, #queue-req: 0 | |
| [2026-06-26 20:05:08 DP1 TP1] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1133.62, #queue-req: 0 | |
| [2026-06-26 20:05:08 DP3 TP3] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1133.81, #queue-req: 0 | |
| [2026-06-26 20:05:08 DP2 TP2] Decode batch, #running-req: 32, #token: 753664, token usage: 0.42, cuda graph: True, gen throughput (token/s): 1133.82, #queue-req: 0 | |
| [2026-06-26 20:05:09 DP0 TP0] Decode batch, #running-req: 32, #token: 755712, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1133.11, #queue-req: 0 | |
| [2026-06-26 20:05:09 DP1 TP1] Decode batch, #running-req: 32, #token: 755712, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1133.15, #queue-req: 0 | |
| [2026-06-26 20:05:09 DP3 TP3] Decode batch, #running-req: 32, #token: 755712, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1133.13, #queue-req: 0 | |
| [2026-06-26 20:05:09 DP2 TP2] Decode batch, #running-req: 32, #token: 755712, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1133.13, #queue-req: 0 | |
| [2026-06-26 20:05:10 DP1 TP1] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1131.31, #queue-req: 0 | |
| [2026-06-26 20:05:10 DP0 TP0] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1131.29, #queue-req: 0 | |
| [2026-06-26 20:05:10 DP3 TP3] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1131.14, #queue-req: 0 | |
| [2026-06-26 20:05:10 DP2 TP2] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1131.14, #queue-req: 0 | |
| [2026-06-26 20:05:11 DP0 TP0] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.71, #queue-req: 0 | |
| [2026-06-26 20:05:11 DP1 TP1] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.71, #queue-req: 0 | |
| [2026-06-26 20:05:11 DP3 TP3] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.74, #queue-req: 0 | |
| [2026-06-26 20:05:11 DP2 TP2] Decode batch, #running-req: 32, #token: 757760, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.74, #queue-req: 0 | |
| [2026-06-26 20:05:12 DP1 TP1] Decode batch, #running-req: 32, #token: 759808, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.88, #queue-req: 0 | |
| [2026-06-26 20:05:12 DP0 TP0] Decode batch, #running-req: 32, #token: 759808, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.87, #queue-req: 0 | |
| [2026-06-26 20:05:12 DP3 TP3] Decode batch, #running-req: 32, #token: 759808, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.80, #queue-req: 0 | |
| [2026-06-26 20:05:12 DP2 TP2] Decode batch, #running-req: 32, #token: 759808, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1130.80, #queue-req: 0 | |
| [2026-06-26 20:05:13 DP0 TP0] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.19, #queue-req: 0 | |
| [2026-06-26 20:05:13 DP1 TP1] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.18, #queue-req: 0 | |
| [2026-06-26 20:05:13 DP3 TP3] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.00, #queue-req: 0 | |
| [2026-06-26 20:05:13 DP2 TP2] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1126.99, #queue-req: 0 | |
| [2026-06-26 20:05:15 DP1 TP1] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1126.57, #queue-req: 0 | |
| [2026-06-26 20:05:15 DP0 TP0] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1126.57, #queue-req: 0 | |
| [2026-06-26 20:05:15 DP3 TP3] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1126.73, #queue-req: 0 | |
| [2026-06-26 20:05:15 DP2 TP2] Decode batch, #running-req: 32, #token: 761856, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1126.74, #queue-req: 0 | |
| [2026-06-26 20:05:16 DP0 TP0] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.91, #queue-req: 0 | |
| [2026-06-26 20:05:16 DP1 TP1] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.90, #queue-req: 0 | |
| [2026-06-26 20:05:16 DP3 TP3] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.80, #queue-req: 0 | |
| [2026-06-26 20:05:16 DP2 TP2] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1127.79, #queue-req: 0 | |
| [2026-06-26 20:05:17 DP1 TP1] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.34, #queue-req: 0 | |
| [2026-06-26 20:05:17 DP0 TP0] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.35, #queue-req: 0 | |
| [2026-06-26 20:05:17 DP3 TP3] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.37, #queue-req: 0 | |
| [2026-06-26 20:05:17 DP2 TP2] Decode batch, #running-req: 32, #token: 763904, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.36, #queue-req: 0 | |
| [2026-06-26 20:05:18 DP0 TP0] Decode batch, #running-req: 32, #token: 765952, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.88, #queue-req: 0 | |
| [2026-06-26 20:05:18 DP1 TP1] Decode batch, #running-req: 32, #token: 765952, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.88, #queue-req: 0 | |
| [2026-06-26 20:05:18 DP3 TP3] Decode batch, #running-req: 32, #token: 765952, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.94, #queue-req: 0 | |
| [2026-06-26 20:05:18 DP2 TP2] Decode batch, #running-req: 32, #token: 765952, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.94, #queue-req: 0 | |
| [2026-06-26 20:05:19 DP1 TP1] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.44, #queue-req: 0 | |
| [2026-06-26 20:05:19 DP0 TP0] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.44, #queue-req: 0 | |
| [2026-06-26 20:05:19 DP3 TP3] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.44, #queue-req: 0 | |
| [2026-06-26 20:05:19 DP2 TP2] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.43, #queue-req: 0 | |
| [2026-06-26 20:05:20 DP1 TP1] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.53, #queue-req: 0 | |
| [2026-06-26 20:05:20 DP0 TP0] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.48, #queue-req: 0 | |
| [2026-06-26 20:05:20 DP3 TP3] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.47, #queue-req: 0 | |
| [2026-06-26 20:05:20 DP2 TP2] Decode batch, #running-req: 32, #token: 768000, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.48, #queue-req: 0 | |
| [2026-06-26 20:05:21 DP1 TP1] Decode batch, #running-req: 32, #token: 770048, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.01, #queue-req: 0 | |
| [2026-06-26 20:05:21 DP0 TP0] Decode batch, #running-req: 32, #token: 770048, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.05, #queue-req: 0 | |
| [2026-06-26 20:05:21 DP2 TP2] Decode batch, #running-req: 32, #token: 770048, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.03, #queue-req: 0 | |
| [2026-06-26 20:05:21 DP3 TP3] Decode batch, #running-req: 32, #token: 770048, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1124.02, #queue-req: 0 | |
| [2026-06-26 20:05:23 DP1 TP1] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.86, #queue-req: 0 | |
| [2026-06-26 20:05:23 DP0 TP0] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.86, #queue-req: 0 | |
| [2026-06-26 20:05:23 DP2 TP2] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.84, #queue-req: 0 | |
| [2026-06-26 20:05:23 DP3 TP3] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.85, #queue-req: 0 | |
| [2026-06-26 20:05:24 DP1 TP1] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.03, #queue-req: 0 | |
| [2026-06-26 20:05:24 DP0 TP0] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.03, #queue-req: 0 | |
| [2026-06-26 20:05:24 DP3 TP3] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.06, #queue-req: 0 | |
| [2026-06-26 20:05:24 DP2 TP2] Decode batch, #running-req: 32, #token: 772096, token usage: 0.43, cuda graph: True, gen throughput (token/s): 1123.07, #queue-req: 0 | |
| [2026-06-26 20:05:25 DP0 TP0] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.99, #queue-req: 0 | |
| [2026-06-26 20:05:25 DP1 TP1] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.99, #queue-req: 0 | |
| [2026-06-26 20:05:25 DP2 TP2] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.92, #queue-req: 0 | |
| [2026-06-26 20:05:25 DP3 TP3] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.92, #queue-req: 0 | |
| [2026-06-26 20:05:26 DP0 TP0] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.80, #queue-req: 0 | |
| [2026-06-26 20:05:26 DP1 TP1] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.80, #queue-req: 0 | |
| [2026-06-26 20:05:26 DP3 TP3] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.80, #queue-req: 0 | |
| [2026-06-26 20:05:26 DP2 TP2] Decode batch, #running-req: 32, #token: 774144, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.79, #queue-req: 0 | |
| [2026-06-26 20:05:27 DP0 TP0] Decode batch, #running-req: 32, #token: 776192, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.78, #queue-req: 0 | |
| [2026-06-26 20:05:27 DP1 TP1] Decode batch, #running-req: 32, #token: 776192, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.77, #queue-req: 0 | |
| [2026-06-26 20:05:27 DP3 TP3] Decode batch, #running-req: 32, #token: 776192, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.84, #queue-req: 0 | |
| [2026-06-26 20:05:27 DP2 TP2] Decode batch, #running-req: 32, #token: 776192, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.83, #queue-req: 0 | |
| [2026-06-26 20:05:28 DP0 TP0] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1125.36, #queue-req: 0 | |
| [2026-06-26 20:05:28 DP1 TP1] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1125.37, #queue-req: 0 | |
| [2026-06-26 20:05:28 DP3 TP3] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1125.36, #queue-req: 0 | |
| [2026-06-26 20:05:28 DP2 TP2] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1125.36, #queue-req: 0 | |
| [2026-06-26 20:05:29 DP0 TP0] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1112.08, #queue-req: 0 | |
| [2026-06-26 20:05:29 DP1 TP1] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1112.08, #queue-req: 0 | |
| [2026-06-26 20:05:29 DP3 TP3] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1112.09, #queue-req: 0 | |
| [2026-06-26 20:05:29 DP2 TP2] Decode batch, #running-req: 32, #token: 778240, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1112.08, #queue-req: 0 | |
| [2026-06-26 20:05:31 DP0 TP0] Decode batch, #running-req: 32, #token: 780288, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.19, #queue-req: 0 | |
| [2026-06-26 20:05:31 DP1 TP1] Decode batch, #running-req: 32, #token: 780288, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.19, #queue-req: 0 | |
| [2026-06-26 20:05:31 DP2 TP2] Decode batch, #running-req: 32, #token: 780288, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.15, #queue-req: 0 | |
| [2026-06-26 20:05:31 DP3 TP3] Decode batch, #running-req: 32, #token: 780288, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.12, #queue-req: 0 | |
| [2026-06-26 20:05:32 DP1 TP1] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.39, #queue-req: 0 | |
| [2026-06-26 20:05:32 DP0 TP0] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.39, #queue-req: 0 | |
| [2026-06-26 20:05:32 DP2 TP2] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.38, #queue-req: 0 | |
| [2026-06-26 20:05:32 DP3 TP3] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.41, #queue-req: 0 | |
| [2026-06-26 20:05:33 DP0 TP0] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.44, #queue-req: 0 | |
| [2026-06-26 20:05:33 DP1 TP1] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.43, #queue-req: 0 | |
| [2026-06-26 20:05:33 DP2 TP2] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.50, #queue-req: 0 | |
| [2026-06-26 20:05:33 DP3 TP3] Decode batch, #running-req: 32, #token: 782336, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.50, #queue-req: 0 | |
| [2026-06-26 20:05:34 DP1 TP1] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.15, #queue-req: 0 | |
| [2026-06-26 20:05:34 DP0 TP0] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.14, #queue-req: 0 | |
| [2026-06-26 20:05:34 DP2 TP2] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.05, #queue-req: 0 | |
| [2026-06-26 20:05:34 DP3 TP3] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.05, #queue-req: 0 | |
| [2026-06-26 20:05:35 DP0 TP0] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.33, #queue-req: 0 | |
| [2026-06-26 20:05:35 DP1 TP1] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.34, #queue-req: 0 | |
| [2026-06-26 20:05:35 DP3 TP3] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.43, #queue-req: 0 | |
| [2026-06-26 20:05:35 DP2 TP2] Decode batch, #running-req: 32, #token: 784384, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1122.43, #queue-req: 0 | |
| [2026-06-26 20:05:36 DP1 TP1] Decode batch, #running-req: 32, #token: 786432, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.21, #queue-req: 0 | |
| [2026-06-26 20:05:36 DP0 TP0] Decode batch, #running-req: 32, #token: 786432, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.21, #queue-req: 0 | |
| [2026-06-26 20:05:36 DP2 TP2] Decode batch, #running-req: 32, #token: 786432, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.17, #queue-req: 0 | |
| [2026-06-26 20:05:36 DP3 TP3] Decode batch, #running-req: 32, #token: 786432, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1121.15, #queue-req: 0 | |
| [2026-06-26 20:05:37 DP1 TP1] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1119.64, #queue-req: 0 | |
| [2026-06-26 20:05:37 DP0 TP0] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1119.64, #queue-req: 0 | |
| [2026-06-26 20:05:37 DP3 TP3] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1119.52, #queue-req: 0 | |
| [2026-06-26 20:05:37 DP2 TP2] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1119.52, #queue-req: 0 | |
| [2026-06-26 20:05:39 DP0 TP0] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.02, #queue-req: 0 | |
| [2026-06-26 20:05:39 DP1 TP1] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.02, #queue-req: 0 | |
| [2026-06-26 20:05:39 DP2 TP2] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.10, #queue-req: 0 | |
| [2026-06-26 20:05:39 DP3 TP3] Decode batch, #running-req: 32, #token: 788480, token usage: 0.44, cuda graph: True, gen throughput (token/s): 1120.11, #queue-req: 0 | |
| [2026-06-26 20:05:40 DP1 TP1] Decode batch, #running-req: 32, #token: 790528, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.41, #queue-req: 0 | |
| [2026-06-26 20:05:40 DP0 TP0] Decode batch, #running-req: 32, #token: 790528, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.41, #queue-req: 0 | |
| [2026-06-26 20:05:40 DP3 TP3] Decode batch, #running-req: 32, #token: 790528, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.42, #queue-req: 0 | |
| [2026-06-26 20:05:40 DP2 TP2] Decode batch, #running-req: 32, #token: 790528, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.44, #queue-req: 0 | |
| [2026-06-26 20:05:41 DP1 TP1] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.07, #queue-req: 0 | |
| [2026-06-26 20:05:41 DP0 TP0] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.07, #queue-req: 0 | |
| [2026-06-26 20:05:41 DP3 TP3] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.07, #queue-req: 0 | |
| [2026-06-26 20:05:41 DP2 TP2] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.07, #queue-req: 0 | |
| [2026-06-26 20:05:42 DP1 TP1] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.55, #queue-req: 0 | |
| [2026-06-26 20:05:42 DP0 TP0] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.53, #queue-req: 0 | |
| [2026-06-26 20:05:42 DP2 TP2] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.41, #queue-req: 0 | |
| [2026-06-26 20:05:42 DP3 TP3] Decode batch, #running-req: 32, #token: 792576, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.41, #queue-req: 0 | |
| [2026-06-26 20:05:43 DP1 TP1] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.90, #queue-req: 0 | |
| [2026-06-26 20:05:43 DP0 TP0] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.91, #queue-req: 0 | |
| [2026-06-26 20:05:43 DP3 TP3] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.92, #queue-req: 0 | |
| [2026-06-26 20:05:43 DP2 TP2] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.93, #queue-req: 0 | |
| [2026-06-26 20:05:44 DP0 TP0] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.06, #queue-req: 0 | |
| [2026-06-26 20:05:44 DP1 TP1] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.06, #queue-req: 0 | |
| [2026-06-26 20:05:44 DP2 TP2] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.13, #queue-req: 0 | |
| [2026-06-26 20:05:44 DP3 TP3] Decode batch, #running-req: 32, #token: 794624, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.13, #queue-req: 0 | |
| [2026-06-26 20:05:45 DP1 TP1] Decode batch, #running-req: 32, #token: 796672, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1121.54, #queue-req: 0 | |
| [2026-06-26 20:05:45 DP0 TP0] Decode batch, #running-req: 32, #token: 796672, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1121.53, #queue-req: 0 | |
| [2026-06-26 20:05:45 DP2 TP2] Decode batch, #running-req: 32, #token: 796672, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1121.53, #queue-req: 0 | |
| [2026-06-26 20:05:45 DP3 TP3] Decode batch, #running-req: 32, #token: 796672, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1121.51, #queue-req: 0 | |
| [2026-06-26 20:05:47 DP1 TP1] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.42, #queue-req: 0 | |
| [2026-06-26 20:05:47 DP0 TP0] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.43, #queue-req: 0 | |
| [2026-06-26 20:05:47 DP2 TP2] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.52, #queue-req: 0 | |
| [2026-06-26 20:05:47 DP3 TP3] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.53, #queue-req: 0 | |
| [2026-06-26 20:05:48 DP1 TP1] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.55, #queue-req: 0 | |
| [2026-06-26 20:05:48 DP0 TP0] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.55, #queue-req: 0 | |
| [2026-06-26 20:05:48 DP2 TP2] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.49, #queue-req: 0 | |
| [2026-06-26 20:05:48 DP3 TP3] Decode batch, #running-req: 32, #token: 798720, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1118.49, #queue-req: 0 | |
| [2026-06-26 20:05:49 DP1 TP1] Decode batch, #running-req: 32, #token: 800768, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.16, #queue-req: 0 | |
| [2026-06-26 20:05:49 DP0 TP0] Decode batch, #running-req: 32, #token: 800768, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.16, #queue-req: 0 | |
| [2026-06-26 20:05:49 DP3 TP3] Decode batch, #running-req: 32, #token: 800768, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.19, #queue-req: 0 | |
| [2026-06-26 20:05:49 DP2 TP2] Decode batch, #running-req: 32, #token: 800768, token usage: 0.45, cuda graph: True, gen throughput (token/s): 1119.17, #queue-req: 0 | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47648 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47694 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47732 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47766 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47786 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47824 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47860 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47914 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47956 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47994 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48020 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48052 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48090 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48116 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48140 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48174 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48204 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48240 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48286 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48314 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48354 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48388 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48440 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48490 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48540 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48584 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48610 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48636 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48660 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48706 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48724 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48750 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47664 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47742 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47772 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47794 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47832 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47870 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47918 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47964 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47998 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48022 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48058 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48094 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48118 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48150 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48176 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48210 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48250 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48298 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48324 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48370 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48394 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48446 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48498 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48554 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48594 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48614 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48638 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48670 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48708 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48736 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48764 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47668 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47704 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47774 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47806 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47846 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47886 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47932 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47980 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48006 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48038 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48072 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48100 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48128 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48160 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48186 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48214 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48266 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48300 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48340 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48376 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48410 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48462 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48514 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48570 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48596 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48626 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48652 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48686 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48714 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48738 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48776 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47682 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47718 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47754 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47784 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47812 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47852 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47900 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47940 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:47992 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48008 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48044 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48086 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48108 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48138 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48164 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48198 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48228 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48280 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48310 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48350 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48382 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48424 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48476 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48524 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48576 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48598 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48630 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48654 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48720 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:49] INFO: 127.0.0.1:48780 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:05:51 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.02, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 152.02 | |
| [2026-06-26 20:05:54 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.04, #running-req: 0, #queue-req: 8, #pending-token: 0, cuda graph: False, input throughput (token/s): 517.52 | |
| [2026-06-26 20:05:54 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.04, #running-req: 0, #queue-req: 7, #pending-token: 0, cuda graph: False, input throughput (token/s): 516.49 | |
| [2026-06-26 20:05:56 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.05, #running-req: 1, #queue-req: 18, #pending-token: 257449, cuda graph: False, input throughput (token/s): 2202.27 | |
| [2026-06-26 20:05:56 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.04, #running-req: 0, #queue-req: 18, #pending-token: 160545, cuda graph: False, input throughput (token/s): 499.44 | |
| [2026-06-26 20:05:56 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.05, #running-req: 1, #queue-req: 17, #pending-token: 226093, cuda graph: False, input throughput (token/s): 15090.03 | |
| [2026-06-26 20:05:56 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.05, #running-req: 1, #queue-req: 17, #pending-token: 256462, cuda graph: False, input throughput (token/s): 4336.95 | |
| [2026-06-26 20:05:58 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.05, #running-req: 1, #queue-req: 27, #pending-token: 577599, cuda graph: False, input throughput (token/s): 15101.91 | |
| [2026-06-26 20:05:58 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.07, #running-req: 2, #queue-req: 27, #pending-token: 579101, cuda graph: False, input throughput (token/s): 4331.93 | |
| [2026-06-26 20:05:58 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.07, #running-req: 2, #queue-req: 27, #pending-token: 545420, cuda graph: False, input throughput (token/s): 4332.23 | |
| [2026-06-26 20:05:58 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32960, #cached-token: 0, token usage: 0.07, #running-req: 2, #queue-req: 27, #pending-token: 547029, cuda graph: False, input throughput (token/s): 15489.55 | |
| [2026-06-26 20:06:00 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.07, #running-req: 2, #queue-req: 28, #pending-token: 867369, cuda graph: False, input throughput (token/s): 14098.01 | |
| [2026-06-26 20:06:00 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9536, #cached-token: 22912, token usage: 0.09, #running-req: 3, #queue-req: 27, #pending-token: 867527, cuda graph: False, input throughput (token/s): 4176.20 | |
| [2026-06-26 20:06:00 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32448, #cached-token: 0, token usage: 0.09, #running-req: 3, #queue-req: 27, #pending-token: 867878, cuda graph: False, input throughput (token/s): 14214.06 | |
| [2026-06-26 20:06:00 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.09, #running-req: 3, #queue-req: 27, #pending-token: 866951, cuda graph: False, input throughput (token/s): 4008.11 | |
| [2026-06-26 20:06:02 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.11, #running-req: 4, #queue-req: 26, #pending-token: 866985, cuda graph: False, input throughput (token/s): 13666.93 | |
| [2026-06-26 20:06:02 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32320, #cached-token: 0, token usage: 0.11, #running-req: 4, #queue-req: 26, #pending-token: 867715, cuda graph: False, input throughput (token/s): 13762.99 | |
| [2026-06-26 20:06:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 22912, token usage: 0.11, #running-req: 4, #queue-req: 26, #pending-token: 867509, cuda graph: False, input throughput (token/s): 3951.48 | |
| [2026-06-26 20:06:02 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.09, #running-req: 3, #queue-req: 27, #pending-token: 899727, cuda graph: False, input throughput (token/s): 13650.17 | |
| [2026-06-26 20:06:05 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.13, #running-req: 5, #queue-req: 25, #pending-token: 835587, cuda graph: False, input throughput (token/s): 13454.87 | |
| [2026-06-26 20:06:05 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.11, #running-req: 4, #queue-req: 26, #pending-token: 867665, cuda graph: False, input throughput (token/s): 13431.33 | |
| [2026-06-26 20:06:05 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.13, #running-req: 5, #queue-req: 25, #pending-token: 835272, cuda graph: False, input throughput (token/s): 13506.67 | |
| [2026-06-26 20:06:05 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.13, #running-req: 5, #queue-req: 25, #pending-token: 834905, cuda graph: False, input throughput (token/s): 3855.18 | |
| [2026-06-26 20:06:07 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.14, #running-req: 6, #queue-req: 24, #pending-token: 802862, cuda graph: False, input throughput (token/s): 13570.45 | |
| [2026-06-26 20:06:07 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9408, #cached-token: 22912, token usage: 0.15, #running-req: 6, #queue-req: 24, #pending-token: 803267, cuda graph: False, input throughput (token/s): 3980.56 | |
| [2026-06-26 20:06:07 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32384, #cached-token: 0, token usage: 0.15, #running-req: 6, #queue-req: 24, #pending-token: 802950, cuda graph: False, input throughput (token/s): 13704.63 | |
| [2026-06-26 20:06:07 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.13, #running-req: 5, #queue-req: 25, #pending-token: 835467, cuda graph: False, input throughput (token/s): 13647.32 | |
| [2026-06-26 20:06:10 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.16, #running-req: 7, #queue-req: 23, #pending-token: 770725, cuda graph: False, input throughput (token/s): 12404.79 | |
| [2026-06-26 20:06:10 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.16, #running-req: 7, #queue-req: 23, #pending-token: 771208, cuda graph: False, input throughput (token/s): 3526.71 | |
| [2026-06-26 20:06:10 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.16, #running-req: 7, #queue-req: 23, #pending-token: 770881, cuda graph: False, input throughput (token/s): 12380.07 | |
| [2026-06-26 20:06:10 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.14, #running-req: 6, #queue-req: 24, #pending-token: 803361, cuda graph: False, input throughput (token/s): 12372.31 | |
| [2026-06-26 20:06:12 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.18, #running-req: 8, #queue-req: 22, #pending-token: 738694, cuda graph: False, input throughput (token/s): 13067.53 | |
| [2026-06-26 20:06:12 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.18, #running-req: 8, #queue-req: 22, #pending-token: 738729, cuda graph: False, input throughput (token/s): 13107.79 | |
| [2026-06-26 20:06:12 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.16, #running-req: 7, #queue-req: 23, #pending-token: 771256, cuda graph: False, input throughput (token/s): 13090.21 | |
| [2026-06-26 20:06:12 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.18, #running-req: 8, #queue-req: 22, #pending-token: 739210, cuda graph: False, input throughput (token/s): 13027.14 | |
| [2026-06-26 20:06:15 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.20, #running-req: 9, #queue-req: 21, #pending-token: 706741, cuda graph: False, input throughput (token/s): 12649.30 | |
| [2026-06-26 20:06:15 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.20, #running-req: 9, #queue-req: 21, #pending-token: 707197, cuda graph: False, input throughput (token/s): 12676.10 | |
| [2026-06-26 20:06:15 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.20, #running-req: 9, #queue-req: 21, #pending-token: 706664, cuda graph: False, input throughput (token/s): 3614.18 | |
| [2026-06-26 20:06:15 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.18, #running-req: 8, #queue-req: 22, #pending-token: 739080, cuda graph: False, input throughput (token/s): 12725.27 | |
| [2026-06-26 20:06:17 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.20, #running-req: 9, #queue-req: 21, #pending-token: 706988, cuda graph: False, input throughput (token/s): 12977.74 | |
| [2026-06-26 20:06:17 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.22, #running-req: 10, #queue-req: 20, #pending-token: 675081, cuda graph: False, input throughput (token/s): 12977.02 | |
| [2026-06-26 20:06:17 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.22, #running-req: 10, #queue-req: 20, #pending-token: 674647, cuda graph: False, input throughput (token/s): 12935.25 | |
| [2026-06-26 20:06:17 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.22, #running-req: 10, #queue-req: 20, #pending-token: 674676, cuda graph: False, input throughput (token/s): 12959.69 | |
| [2026-06-26 20:06:20 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32320, #cached-token: 0, token usage: 0.24, #running-req: 11, #queue-req: 19, #pending-token: 642371, cuda graph: False, input throughput (token/s): 12772.35 | |
| [2026-06-26 20:06:20 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.22, #running-req: 10, #queue-req: 20, #pending-token: 674958, cuda graph: False, input throughput (token/s): 12654.56 | |
| [2026-06-26 20:06:20 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.24, #running-req: 11, #queue-req: 19, #pending-token: 642544, cuda graph: False, input throughput (token/s): 12695.26 | |
| [2026-06-26 20:06:20 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.24, #running-req: 11, #queue-req: 19, #pending-token: 642978, cuda graph: False, input throughput (token/s): 3636.57 | |
| [2026-06-26 20:06:22 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.25, #running-req: 12, #queue-req: 18, #pending-token: 610256, cuda graph: False, input throughput (token/s): 13097.27 | |
| [2026-06-26 20:06:22 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.25, #running-req: 12, #queue-req: 18, #pending-token: 610874, cuda graph: False, input throughput (token/s): 13100.27 | |
| [2026-06-26 20:06:22 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.24, #running-req: 11, #queue-req: 19, #pending-token: 642704, cuda graph: False, input throughput (token/s): 13147.20 | |
| [2026-06-26 20:06:22 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.25, #running-req: 12, #queue-req: 18, #pending-token: 610336, cuda graph: False, input throughput (token/s): 13147.05 | |
| [2026-06-26 20:06:24 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.27, #running-req: 13, #queue-req: 17, #pending-token: 578218, cuda graph: False, input throughput (token/s): 14680.56 | |
| [2026-06-26 20:06:24 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.27, #running-req: 13, #queue-req: 17, #pending-token: 578271, cuda graph: False, input throughput (token/s): 14713.15 | |
| [2026-06-26 20:06:24 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.25, #running-req: 12, #queue-req: 18, #pending-token: 610714, cuda graph: False, input throughput (token/s): 14653.62 | |
| [2026-06-26 20:06:24 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 22912, token usage: 0.27, #running-req: 13, #queue-req: 17, #pending-token: 578637, cuda graph: False, input throughput (token/s): 4277.91 | |
| [2026-06-26 20:06:27 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.29, #running-req: 14, #queue-req: 16, #pending-token: 546209, cuda graph: False, input throughput (token/s): 13177.36 | |
| [2026-06-26 20:06:27 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.27, #running-req: 13, #queue-req: 17, #pending-token: 578717, cuda graph: False, input throughput (token/s): 13151.01 | |
| [2026-06-26 20:06:27 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 22912, token usage: 0.29, #running-req: 14, #queue-req: 16, #pending-token: 546402, cuda graph: False, input throughput (token/s): 3839.84 | |
| [2026-06-26 20:06:27 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.29, #running-req: 14, #queue-req: 16, #pending-token: 546146, cuda graph: False, input throughput (token/s): 3787.10 | |
| [2026-06-26 20:06:29 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.31, #running-req: 15, #queue-req: 15, #pending-token: 514157, cuda graph: False, input throughput (token/s): 13795.21 | |
| [2026-06-26 20:06:29 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.31, #running-req: 15, #queue-req: 15, #pending-token: 514001, cuda graph: False, input throughput (token/s): 13851.59 | |
| [2026-06-26 20:06:29 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.29, #running-req: 14, #queue-req: 16, #pending-token: 546612, cuda graph: False, input throughput (token/s): 13819.45 | |
| [2026-06-26 20:06:29 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32832, #cached-token: 0, token usage: 0.31, #running-req: 15, #queue-req: 15, #pending-token: 513622, cuda graph: False, input throughput (token/s): 14121.92 | |
| [2026-06-26 20:06:31 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.33, #running-req: 16, #queue-req: 14, #pending-token: 481933, cuda graph: False, input throughput (token/s): 15347.52 | |
| [2026-06-26 20:06:31 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.33, #running-req: 16, #queue-req: 14, #pending-token: 481581, cuda graph: False, input throughput (token/s): 15323.22 | |
| [2026-06-26 20:06:31 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9984, #cached-token: 22912, token usage: 0.33, #running-req: 16, #queue-req: 14, #pending-token: 481321, cuda graph: False, input throughput (token/s): 4769.14 | |
| [2026-06-26 20:06:31 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.31, #running-req: 15, #queue-req: 15, #pending-token: 514562, cuda graph: False, input throughput (token/s): 4372.95 | |
| [2026-06-26 20:06:33 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 22912, token usage: 0.34, #running-req: 17, #queue-req: 13, #pending-token: 449426, cuda graph: False, input throughput (token/s): 4495.80 | |
| [2026-06-26 20:06:33 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.33, #running-req: 16, #queue-req: 14, #pending-token: 482442, cuda graph: False, input throughput (token/s): 4464.54 | |
| [2026-06-26 20:06:33 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.34, #running-req: 17, #queue-req: 13, #pending-token: 449844, cuda graph: False, input throughput (token/s): 15542.96 | |
| [2026-06-26 20:06:33 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.34, #running-req: 17, #queue-req: 13, #pending-token: 449180, cuda graph: False, input throughput (token/s): 15572.37 | |
| [2026-06-26 20:06:35 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.36, #running-req: 18, #queue-req: 12, #pending-token: 417071, cuda graph: False, input throughput (token/s): 16790.01 | |
| [2026-06-26 20:06:35 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.34, #running-req: 17, #queue-req: 13, #pending-token: 450386, cuda graph: False, input throughput (token/s): 4775.27 | |
| [2026-06-26 20:06:35 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.36, #running-req: 18, #queue-req: 12, #pending-token: 417398, cuda graph: False, input throughput (token/s): 4774.63 | |
| [2026-06-26 20:06:35 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.36, #running-req: 18, #queue-req: 12, #pending-token: 417789, cuda graph: False, input throughput (token/s): 16736.08 | |
| [2026-06-26 20:06:37 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.38, #running-req: 19, #queue-req: 11, #pending-token: 385022, cuda graph: False, input throughput (token/s): 4189.90 | |
| [2026-06-26 20:06:37 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.36, #running-req: 18, #queue-req: 12, #pending-token: 418279, cuda graph: False, input throughput (token/s): 4219.29 | |
| [2026-06-26 20:06:37 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.38, #running-req: 19, #queue-req: 11, #pending-token: 385598, cuda graph: False, input throughput (token/s): 14752.03 | |
| [2026-06-26 20:06:37 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.38, #running-req: 19, #queue-req: 11, #pending-token: 385365, cuda graph: False, input throughput (token/s): 4189.99 | |
| [2026-06-26 20:06:40 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.40, #running-req: 20, #queue-req: 10, #pending-token: 353008, cuda graph: False, input throughput (token/s): 12679.51 | |
| [2026-06-26 20:06:40 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.38, #running-req: 19, #queue-req: 11, #pending-token: 386310, cuda graph: False, input throughput (token/s): 12654.00 | |
| [2026-06-26 20:06:40 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.40, #running-req: 20, #queue-req: 10, #pending-token: 353568, cuda graph: False, input throughput (token/s): 3618.94 | |
| [2026-06-26 20:06:40 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.40, #running-req: 20, #queue-req: 10, #pending-token: 353304, cuda graph: False, input throughput (token/s): 12679.25 | |
| [2026-06-26 20:06:43 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.40, #running-req: 20, #queue-req: 10, #pending-token: 354114, cuda graph: False, input throughput (token/s): 12392.44 | |
| [2026-06-26 20:06:43 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.42, #running-req: 21, #queue-req: 9, #pending-token: 320898, cuda graph: False, input throughput (token/s): 12336.23 | |
| [2026-06-26 20:06:43 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.42, #running-req: 21, #queue-req: 9, #pending-token: 321412, cuda graph: False, input throughput (token/s): 12361.39 | |
| [2026-06-26 20:06:43 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.42, #running-req: 21, #queue-req: 9, #pending-token: 321303, cuda graph: False, input throughput (token/s): 12312.35 | |
| [2026-06-26 20:06:45 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.43, #running-req: 22, #queue-req: 8, #pending-token: 289304, cuda graph: False, input throughput (token/s): 13412.14 | |
| [2026-06-26 20:06:45 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.42, #running-req: 21, #queue-req: 9, #pending-token: 322061, cuda graph: False, input throughput (token/s): 3818.27 | |
| [2026-06-26 20:06:45 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.43, #running-req: 22, #queue-req: 8, #pending-token: 288812, cuda graph: False, input throughput (token/s): 13408.45 | |
| [2026-06-26 20:06:45 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32768, #cached-token: 0, token usage: 0.44, #running-req: 22, #queue-req: 8, #pending-token: 288568, cuda graph: False, input throughput (token/s): 13674.22 | |
| [2026-06-26 20:06:47 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.45, #running-req: 23, #queue-req: 7, #pending-token: 257215, cuda graph: False, input throughput (token/s): 3897.89 | |
| [2026-06-26 20:06:47 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.45, #running-req: 23, #queue-req: 7, #pending-token: 256771, cuda graph: False, input throughput (token/s): 13564.81 | |
| [2026-06-26 20:06:47 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.43, #running-req: 22, #queue-req: 8, #pending-token: 290023, cuda graph: False, input throughput (token/s): 13561.32 | |
| [2026-06-26 20:06:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.45, #running-req: 23, #queue-req: 7, #pending-token: 256453, cuda graph: False, input throughput (token/s): 13593.16 | |
| [2026-06-26 20:06:50 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32512, #cached-token: 0, token usage: 0.47, #running-req: 24, #queue-req: 6, #pending-token: 224753, cuda graph: False, input throughput (token/s): 13749.72 | |
| [2026-06-26 20:06:50 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.47, #running-req: 24, #queue-req: 6, #pending-token: 224434, cuda graph: False, input throughput (token/s): 13560.43 | |
| [2026-06-26 20:06:50 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.47, #running-req: 24, #queue-req: 6, #pending-token: 224697, cuda graph: False, input throughput (token/s): 13586.86 | |
| [2026-06-26 20:06:50 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 22912, token usage: 0.45, #running-req: 23, #queue-req: 7, #pending-token: 257953, cuda graph: False, input throughput (token/s): 3897.39 | |
| [2026-06-26 20:06:52 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.49, #running-req: 25, #queue-req: 5, #pending-token: 192584, cuda graph: False, input throughput (token/s): 12306.61 | |
| [2026-06-26 20:06:52 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 22912, token usage: 0.47, #running-req: 24, #queue-req: 6, #pending-token: 225823, cuda graph: False, input throughput (token/s): 3554.63 | |
| [2026-06-26 20:06:52 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.49, #running-req: 25, #queue-req: 5, #pending-token: 192539, cuda graph: False, input throughput (token/s): 12354.58 | |
| [2026-06-26 20:06:52 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.49, #running-req: 25, #queue-req: 5, #pending-token: 192333, cuda graph: False, input throughput (token/s): 12305.65 | |
| [2026-06-26 20:06:55 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.51, #running-req: 26, #queue-req: 4, #pending-token: 160320, cuda graph: False, input throughput (token/s): 12111.85 | |
| [2026-06-26 20:06:55 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.51, #running-req: 26, #queue-req: 4, #pending-token: 160243, cuda graph: False, input throughput (token/s): 12063.93 | |
| [2026-06-26 20:06:55 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.51, #running-req: 26, #queue-req: 4, #pending-token: 160506, cuda graph: False, input throughput (token/s): 12063.11 | |
| [2026-06-26 20:06:55 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 33024, #cached-token: 0, token usage: 0.49, #running-req: 25, #queue-req: 5, #pending-token: 192860, cuda graph: False, input throughput (token/s): 12399.66 | |
| [2026-06-26 20:06:57 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.52, #running-req: 27, #queue-req: 3, #pending-token: 128114, cuda graph: False, input throughput (token/s): 14116.28 | |
| [2026-06-26 20:06:57 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32000, #cached-token: 0, token usage: 0.53, #running-req: 27, #queue-req: 3, #pending-token: 128287, cuda graph: False, input throughput (token/s): 14003.90 | |
| [2026-06-26 20:06:57 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.53, #running-req: 27, #queue-req: 3, #pending-token: 128426, cuda graph: False, input throughput (token/s): 14060.28 | |
| [2026-06-26 20:06:57 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32256, #cached-token: 0, token usage: 0.51, #running-req: 26, #queue-req: 4, #pending-token: 160615, cuda graph: False, input throughput (token/s): 14116.13 | |
| [2026-06-26 20:06:59 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9088, #cached-token: 22912, token usage: 0.54, #running-req: 28, #queue-req: 2, #pending-token: 96131, cuda graph: False, input throughput (token/s): 4007.64 | |
| [2026-06-26 20:06:59 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.54, #running-req: 28, #queue-req: 2, #pending-token: 96231, cuda graph: False, input throughput (token/s): 4035.87 | |
| [2026-06-26 20:06:59 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.53, #running-req: 27, #queue-req: 3, #pending-token: 128558, cuda graph: False, input throughput (token/s): 14139.97 | |
| [2026-06-26 20:06:59 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.54, #running-req: 28, #queue-req: 2, #pending-token: 96264, cuda graph: False, input throughput (token/s): 14195.98 | |
| [2026-06-26 20:07:02 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.56, #running-req: 29, #queue-req: 1, #pending-token: 64155, cuda graph: False, input throughput (token/s): 13744.79 | |
| [2026-06-26 20:07:02 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.56, #running-req: 29, #queue-req: 1, #pending-token: 64085, cuda graph: False, input throughput (token/s): 13705.50 | |
| [2026-06-26 20:07:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.56, #running-req: 29, #queue-req: 1, #pending-token: 64206, cuda graph: False, input throughput (token/s): 13705.03 | |
| [2026-06-26 20:07:02 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 22912, token usage: 0.54, #running-req: 28, #queue-req: 2, #pending-token: 96410, cuda graph: False, input throughput (token/s): 3966.29 | |
| [2026-06-26 20:07:04 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.58, #running-req: 30, #queue-req: 0, #pending-token: 32147, cuda graph: False, input throughput (token/s): 13460.27 | |
| [2026-06-26 20:07:04 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.58, #running-req: 30, #queue-req: 0, #pending-token: 32058, cuda graph: False, input throughput (token/s): 13459.07 | |
| [2026-06-26 20:07:04 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 22912, token usage: 0.58, #running-req: 30, #queue-req: 0, #pending-token: 32107, cuda graph: False, input throughput (token/s): 3838.33 | |
| [2026-06-26 20:07:04 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.56, #running-req: 29, #queue-req: 1, #pending-token: 64367, cuda graph: False, input throughput (token/s): 13460.33 | |
| [2026-06-26 20:07:05 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 22912, token usage: 0.58, #running-req: 30, #queue-req: 0, #pending-token: 32158, cuda graph: False, input throughput (token/s): 8238.53 | |
| [2026-06-26 20:07:05 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 32064, #cached-token: 0, token usage: 0.58, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 27982.20 | |
| [2026-06-26 20:07:05 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 32128, #cached-token: 0, token usage: 0.58, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 28023.34 | |
| [2026-06-26 20:07:05 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 32192, #cached-token: 0, token usage: 0.58, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 28073.70 | |
| [2026-06-26 20:07:05 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 22912, token usage: 0.58, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 55078.61 | |
| [2026-06-26 20:07:06 DP1 TP1] Decode batch, #running-req: 32, #token: 1030272, token usage: 0.58, cuda graph: True, gen throughput (token/s): 16.17, #queue-req: 0 | |
| [2026-06-26 20:07:06 DP0 TP0] Decode batch, #running-req: 32, #token: 1030144, token usage: 0.58, cuda graph: True, gen throughput (token/s): 16.56, #queue-req: 0 | |
| [2026-06-26 20:07:06 DP2 TP2] Decode batch, #running-req: 32, #token: 1029120, token usage: 0.58, cuda graph: True, gen throughput (token/s): 16.57, #queue-req: 0 | |
| [2026-06-26 20:07:06 DP3 TP3] Decode batch, #running-req: 32, #token: 1031296, token usage: 0.58, cuda graph: True, gen throughput (token/s): 16.57, #queue-req: 0 | |
| [2026-06-26 20:07:07 DP1 TP1] Decode batch, #running-req: 32, #token: 1031360, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1149.45, #queue-req: 0 | |
| [2026-06-26 20:07:07 DP2 TP2] Decode batch, #running-req: 32, #token: 1030528, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1149.24, #queue-req: 0 | |
| [2026-06-26 20:07:07 DP3 TP3] Decode batch, #running-req: 32, #token: 1032512, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1149.23, #queue-req: 0 | |
| [2026-06-26 20:07:07 DP0 TP0] Decode batch, #running-req: 32, #token: 1031040, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1149.20, #queue-req: 0 | |
| [2026-06-26 20:07:08 DP1 TP1] Decode batch, #running-req: 32, #token: 1032512, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1143.28, #queue-req: 0 | |
| [2026-06-26 20:07:08 DP3 TP3] Decode batch, #running-req: 32, #token: 1033856, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1143.03, #queue-req: 0 | |
| [2026-06-26 20:07:08 DP2 TP2] Decode batch, #running-req: 32, #token: 1031808, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1143.03, #queue-req: 0 | |
| [2026-06-26 20:07:08 DP0 TP0] Decode batch, #running-req: 32, #token: 1032576, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1143.05, #queue-req: 0 | |
| [2026-06-26 20:07:09 DP1 TP1] Decode batch, #running-req: 32, #token: 1033984, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1137.49, #queue-req: 0 | |
| [2026-06-26 20:07:09 DP3 TP3] Decode batch, #running-req: 32, #token: 1035008, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1137.48, #queue-req: 0 | |
| [2026-06-26 20:07:09 DP2 TP2] Decode batch, #running-req: 32, #token: 1032832, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1137.47, #queue-req: 0 | |
| [2026-06-26 20:07:09 DP0 TP0] Decode batch, #running-req: 32, #token: 1033792, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1137.47, #queue-req: 0 | |
| [2026-06-26 20:07:11 DP1 TP1] Decode batch, #running-req: 32, #token: 1035200, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1133.62, #queue-req: 0 | |
| [2026-06-26 20:07:11 DP2 TP2] Decode batch, #running-req: 32, #token: 1034432, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1133.35, #queue-req: 0 | |
| [2026-06-26 20:07:11 DP3 TP3] Decode batch, #running-req: 32, #token: 1036416, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1133.35, #queue-req: 0 | |
| [2026-06-26 20:07:11 DP0 TP0] Decode batch, #running-req: 32, #token: 1034816, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1133.36, #queue-req: 0 | |
| [2026-06-26 20:07:12 DP1 TP1] Decode batch, #running-req: 32, #token: 1036608, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1130.28, #queue-req: 0 | |
| [2026-06-26 20:07:12 DP2 TP2] Decode batch, #running-req: 32, #token: 1035328, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1130.27, #queue-req: 0 | |
| [2026-06-26 20:07:12 DP3 TP3] Decode batch, #running-req: 32, #token: 1037632, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1130.27, #queue-req: 0 | |
| [2026-06-26 20:07:12 DP0 TP0] Decode batch, #running-req: 32, #token: 1036480, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1130.26, #queue-req: 0 | |
| [2026-06-26 20:07:13 DP1 TP1] Decode batch, #running-req: 32, #token: 1037632, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1125.40, #queue-req: 0 | |
| [2026-06-26 20:07:13 DP2 TP2] Decode batch, #running-req: 32, #token: 1036864, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1125.24, #queue-req: 0 | |
| [2026-06-26 20:07:13 DP3 TP3] Decode batch, #running-req: 32, #token: 1038912, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1125.24, #queue-req: 0 | |
| [2026-06-26 20:07:13 DP0 TP0] Decode batch, #running-req: 32, #token: 1037440, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1125.24, #queue-req: 0 | |
| [2026-06-26 20:07:14 DP1 TP1] Decode batch, #running-req: 32, #token: 1038912, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.04, #queue-req: 0 | |
| [2026-06-26 20:07:14 DP0 TP0] Decode batch, #running-req: 32, #token: 1038848, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.16, #queue-req: 0 | |
| [2026-06-26 20:07:14 DP3 TP3] Decode batch, #running-req: 32, #token: 1040192, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.15, #queue-req: 0 | |
| [2026-06-26 20:07:14 DP2 TP2] Decode batch, #running-req: 32, #token: 1038272, token usage: 0.58, cuda graph: True, gen throughput (token/s): 1124.14, #queue-req: 0 | |
| [2026-06-26 20:07:15 DP1 TP1] Decode batch, #running-req: 32, #token: 1040512, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.54, #queue-req: 0 | |
| [2026-06-26 20:07:15 DP3 TP3] Decode batch, #running-req: 32, #token: 1041536, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.53, #queue-req: 0 | |
| [2026-06-26 20:07:15 DP2 TP2] Decode batch, #running-req: 32, #token: 1039360, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.52, #queue-req: 0 | |
| [2026-06-26 20:07:15 DP0 TP0] Decode batch, #running-req: 32, #token: 1040384, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.51, #queue-req: 0 | |
| [2026-06-26 20:07:16 DP1 TP1] Decode batch, #running-req: 32, #token: 1041600, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.97, #queue-req: 0 | |
| [2026-06-26 20:07:16 DP3 TP3] Decode batch, #running-req: 32, #token: 1042752, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.01, #queue-req: 0 | |
| [2026-06-26 20:07:16 DP2 TP2] Decode batch, #running-req: 32, #token: 1040768, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.01, #queue-req: 0 | |
| [2026-06-26 20:07:16 DP0 TP0] Decode batch, #running-req: 32, #token: 1041280, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.02, #queue-req: 0 | |
| [2026-06-26 20:07:17 DP1 TP1] Decode batch, #running-req: 32, #token: 1042752, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.82, #queue-req: 0 | |
| [2026-06-26 20:07:17 DP3 TP3] Decode batch, #running-req: 32, #token: 1044096, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.75, #queue-req: 0 | |
| [2026-06-26 20:07:17 DP2 TP2] Decode batch, #running-req: 32, #token: 1042048, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.76, #queue-req: 0 | |
| [2026-06-26 20:07:17 DP0 TP0] Decode batch, #running-req: 32, #token: 1042816, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.74, #queue-req: 0 | |
| [2026-06-26 20:07:19 DP1 TP1] Decode batch, #running-req: 32, #token: 1044224, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.57, #queue-req: 0 | |
| [2026-06-26 20:07:19 DP2 TP2] Decode batch, #running-req: 32, #token: 1043072, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.47, #queue-req: 0 | |
| [2026-06-26 20:07:19 DP3 TP3] Decode batch, #running-req: 32, #token: 1045248, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.48, #queue-req: 0 | |
| [2026-06-26 20:07:19 DP0 TP0] Decode batch, #running-req: 32, #token: 1044032, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.48, #queue-req: 0 | |
| [2026-06-26 20:07:20 DP1 TP1] Decode batch, #running-req: 32, #token: 1045440, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.45, #queue-req: 0 | |
| [2026-06-26 20:07:20 DP0 TP0] Decode batch, #running-req: 32, #token: 1045056, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.58, #queue-req: 0 | |
| [2026-06-26 20:07:20 DP3 TP3] Decode batch, #running-req: 32, #token: 1046656, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.56, #queue-req: 0 | |
| [2026-06-26 20:07:20 DP2 TP2] Decode batch, #running-req: 32, #token: 1044672, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.57, #queue-req: 0 | |
| [2026-06-26 20:07:21 DP1 TP1] Decode batch, #running-req: 32, #token: 1046848, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.60, #queue-req: 0 | |
| [2026-06-26 20:07:21 DP0 TP0] Decode batch, #running-req: 32, #token: 1046720, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.43, #queue-req: 0 | |
| [2026-06-26 20:07:21 DP2 TP2] Decode batch, #running-req: 32, #token: 1045568, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.44, #queue-req: 0 | |
| [2026-06-26 20:07:21 DP3 TP3] Decode batch, #running-req: 32, #token: 1047872, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1123.45, #queue-req: 0 | |
| [2026-06-26 20:07:22 DP1 TP1] Decode batch, #running-req: 32, #token: 1047872, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.33, #queue-req: 0 | |
| [2026-06-26 20:07:22 DP0 TP0] Decode batch, #running-req: 32, #token: 1047680, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.29, #queue-req: 0 | |
| [2026-06-26 20:07:22 DP2 TP2] Decode batch, #running-req: 32, #token: 1047104, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.30, #queue-req: 0 | |
| [2026-06-26 20:07:22 DP3 TP3] Decode batch, #running-req: 32, #token: 1049152, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.30, #queue-req: 0 | |
| [2026-06-26 20:07:23 DP1 TP1] Decode batch, #running-req: 32, #token: 1049152, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.50, #queue-req: 0 | |
| [2026-06-26 20:07:23 DP3 TP3] Decode batch, #running-req: 32, #token: 1050432, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.68, #queue-req: 0 | |
| [2026-06-26 20:07:23 DP2 TP2] Decode batch, #running-req: 32, #token: 1048512, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.68, #queue-req: 0 | |
| [2026-06-26 20:07:23 DP0 TP0] Decode batch, #running-req: 32, #token: 1049088, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.67, #queue-req: 0 | |
| [2026-06-26 20:07:24 DP1 TP1] Decode batch, #running-req: 32, #token: 1050752, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.54, #queue-req: 0 | |
| [2026-06-26 20:07:24 DP3 TP3] Decode batch, #running-req: 32, #token: 1051776, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.43, #queue-req: 0 | |
| [2026-06-26 20:07:24 DP2 TP2] Decode batch, #running-req: 32, #token: 1049600, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.42, #queue-req: 0 | |
| [2026-06-26 20:07:24 DP0 TP0] Decode batch, #running-req: 32, #token: 1050624, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1124.43, #queue-req: 0 | |
| [2026-06-26 20:07:25 DP1 TP1] Decode batch, #running-req: 32, #token: 1051840, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.55, #queue-req: 0 | |
| [2026-06-26 20:07:25 DP3 TP3] Decode batch, #running-req: 32, #token: 1052992, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.54, #queue-req: 0 | |
| [2026-06-26 20:07:25 DP2 TP2] Decode batch, #running-req: 32, #token: 1051008, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.55, #queue-req: 0 | |
| [2026-06-26 20:07:25 DP0 TP0] Decode batch, #running-req: 32, #token: 1051520, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1122.53, #queue-req: 0 | |
| [2026-06-26 20:07:27 DP1 TP1] Decode batch, #running-req: 32, #token: 1052992, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.80, #queue-req: 0 | |
| [2026-06-26 20:07:27 DP3 TP3] Decode batch, #running-req: 32, #token: 1054336, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.83, #queue-req: 0 | |
| [2026-06-26 20:07:27 DP2 TP2] Decode batch, #running-req: 32, #token: 1052288, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.83, #queue-req: 0 | |
| [2026-06-26 20:07:27 DP0 TP0] Decode batch, #running-req: 32, #token: 1053056, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.84, #queue-req: 0 | |
| [2026-06-26 20:07:28 DP1 TP1] Decode batch, #running-req: 32, #token: 1054464, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.47, #queue-req: 0 | |
| [2026-06-26 20:07:28 DP2 TP2] Decode batch, #running-req: 32, #token: 1053312, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.51, #queue-req: 0 | |
| [2026-06-26 20:07:28 DP3 TP3] Decode batch, #running-req: 32, #token: 1055488, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.50, #queue-req: 0 | |
| [2026-06-26 20:07:28 DP0 TP0] Decode batch, #running-req: 32, #token: 1054272, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.48, #queue-req: 0 | |
| [2026-06-26 20:07:29 DP1 TP1] Decode batch, #running-req: 32, #token: 1055680, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.84, #queue-req: 0 | |
| [2026-06-26 20:07:29 DP3 TP3] Decode batch, #running-req: 32, #token: 1056896, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.74, #queue-req: 0 | |
| [2026-06-26 20:07:29 DP2 TP2] Decode batch, #running-req: 32, #token: 1054912, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.74, #queue-req: 0 | |
| [2026-06-26 20:07:29 DP0 TP0] Decode batch, #running-req: 32, #token: 1055296, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.77, #queue-req: 0 | |
| [2026-06-26 20:07:30 DP1 TP1] Decode batch, #running-req: 32, #token: 1057088, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.38, #queue-req: 0 | |
| [2026-06-26 20:07:30 DP3 TP3] Decode batch, #running-req: 32, #token: 1058112, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.43, #queue-req: 0 | |
| [2026-06-26 20:07:30 DP2 TP2] Decode batch, #running-req: 32, #token: 1055808, token usage: 0.59, cuda graph: True, gen throughput (token/s): 1120.42, #queue-req: 0 | |
| [2026-06-26 20:07:30 DP0 TP0] Decode batch, #running-req: 32, #token: 1056960, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.41, #queue-req: 0 | |
| [2026-06-26 20:07:31 DP1 TP1] Decode batch, #running-req: 32, #token: 1058112, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.59, #queue-req: 0 | |
| [2026-06-26 20:07:31 DP2 TP2] Decode batch, #running-req: 32, #token: 1057344, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.58, #queue-req: 0 | |
| [2026-06-26 20:07:31 DP3 TP3] Decode batch, #running-req: 32, #token: 1059392, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.57, #queue-req: 0 | |
| [2026-06-26 20:07:31 DP0 TP0] Decode batch, #running-req: 32, #token: 1057920, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1120.58, #queue-req: 0 | |
| [2026-06-26 20:07:32 DP1 TP1] Decode batch, #running-req: 32, #token: 1059392, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.21, #queue-req: 0 | |
| [2026-06-26 20:07:32 DP3 TP3] Decode batch, #running-req: 32, #token: 1060672, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.24, #queue-req: 0 | |
| [2026-06-26 20:07:32 DP2 TP2] Decode batch, #running-req: 32, #token: 1058752, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.23, #queue-req: 0 | |
| [2026-06-26 20:07:32 DP0 TP0] Decode batch, #running-req: 32, #token: 1059328, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.22, #queue-req: 0 | |
| [2026-06-26 20:07:33 DP1 TP1] Decode batch, #running-req: 32, #token: 1060992, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1119.04, #queue-req: 0 | |
| [2026-06-26 20:07:33 DP3 TP3] Decode batch, #running-req: 32, #token: 1062016, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.90, #queue-req: 0 | |
| [2026-06-26 20:07:33 DP2 TP2] Decode batch, #running-req: 32, #token: 1059840, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.91, #queue-req: 0 | |
| [2026-06-26 20:07:33 DP0 TP0] Decode batch, #running-req: 32, #token: 1060864, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.92, #queue-req: 0 | |
| [2026-06-26 20:07:35 DP1 TP1] Decode batch, #running-req: 32, #token: 1062080, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.71, #queue-req: 0 | |
| [2026-06-26 20:07:35 DP3 TP3] Decode batch, #running-req: 32, #token: 1063232, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.72, #queue-req: 0 | |
| [2026-06-26 20:07:35 DP2 TP2] Decode batch, #running-req: 32, #token: 1061248, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.72, #queue-req: 0 | |
| [2026-06-26 20:07:35 DP0 TP0] Decode batch, #running-req: 32, #token: 1061760, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.72, #queue-req: 0 | |
| [2026-06-26 20:07:36 DP1 TP1] Decode batch, #running-req: 32, #token: 1063232, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.92, #queue-req: 0 | |
| [2026-06-26 20:07:36 DP2 TP2] Decode batch, #running-req: 32, #token: 1062528, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.87, #queue-req: 0 | |
| [2026-06-26 20:07:36 DP3 TP3] Decode batch, #running-req: 32, #token: 1064576, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.87, #queue-req: 0 | |
| [2026-06-26 20:07:36 DP0 TP0] Decode batch, #running-req: 32, #token: 1063296, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.87, #queue-req: 0 | |
| [2026-06-26 20:07:37 DP1 TP1] Decode batch, #running-req: 32, #token: 1064704, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1116.95, #queue-req: 0 | |
| [2026-06-26 20:07:37 DP2 TP2] Decode batch, #running-req: 32, #token: 1063552, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.11, #queue-req: 0 | |
| [2026-06-26 20:07:37 DP3 TP3] Decode batch, #running-req: 32, #token: 1065728, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.11, #queue-req: 0 | |
| [2026-06-26 20:07:37 DP0 TP0] Decode batch, #running-req: 32, #token: 1064512, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.08, #queue-req: 0 | |
| [2026-06-26 20:07:38 DP1 TP1] Decode batch, #running-req: 32, #token: 1065920, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.43, #queue-req: 0 | |
| [2026-06-26 20:07:38 DP3 TP3] Decode batch, #running-req: 32, #token: 1067136, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.30, #queue-req: 0 | |
| [2026-06-26 20:07:38 DP2 TP2] Decode batch, #running-req: 32, #token: 1065152, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.29, #queue-req: 0 | |
| [2026-06-26 20:07:38 DP0 TP0] Decode batch, #running-req: 32, #token: 1065536, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.32, #queue-req: 0 | |
| [2026-06-26 20:07:39 DP1 TP1] Decode batch, #running-req: 32, #token: 1067328, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.40, #queue-req: 0 | |
| [2026-06-26 20:07:39 DP3 TP3] Decode batch, #running-req: 32, #token: 1068352, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.39, #queue-req: 0 | |
| [2026-06-26 20:07:39 DP2 TP2] Decode batch, #running-req: 32, #token: 1066048, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.40, #queue-req: 0 | |
| [2026-06-26 20:07:39 DP0 TP0] Decode batch, #running-req: 32, #token: 1067200, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.39, #queue-req: 0 | |
| [2026-06-26 20:07:40 DP1 TP1] Decode batch, #running-req: 32, #token: 1068352, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.77, #queue-req: 0 | |
| [2026-06-26 20:07:40 DP3 TP3] Decode batch, #running-req: 32, #token: 1069632, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.79, #queue-req: 0 | |
| [2026-06-26 20:07:40 DP2 TP2] Decode batch, #running-req: 32, #token: 1067584, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.79, #queue-req: 0 | |
| [2026-06-26 20:07:40 DP0 TP0] Decode batch, #running-req: 32, #token: 1068160, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.79, #queue-req: 0 | |
| [2026-06-26 20:07:41 DP1 TP1] Decode batch, #running-req: 32, #token: 1069632, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.41, #queue-req: 0 | |
| [2026-06-26 20:07:41 DP3 TP3] Decode batch, #running-req: 32, #token: 1070912, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.40, #queue-req: 0 | |
| [2026-06-26 20:07:41 DP2 TP2] Decode batch, #running-req: 32, #token: 1068992, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.41, #queue-req: 0 | |
| [2026-06-26 20:07:41 DP0 TP0] Decode batch, #running-req: 32, #token: 1069568, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.42, #queue-req: 0 | |
| [2026-06-26 20:07:43 DP1 TP1] Decode batch, #running-req: 32, #token: 1071232, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.32, #queue-req: 0 | |
| [2026-06-26 20:07:43 DP3 TP3] Decode batch, #running-req: 32, #token: 1072256, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.43, #queue-req: 0 | |
| [2026-06-26 20:07:43 DP2 TP2] Decode batch, #running-req: 32, #token: 1070080, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.43, #queue-req: 0 | |
| [2026-06-26 20:07:43 DP0 TP0] Decode batch, #running-req: 32, #token: 1071104, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.45, #queue-req: 0 | |
| [2026-06-26 20:07:44 DP1 TP1] Decode batch, #running-req: 32, #token: 1072320, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.66, #queue-req: 0 | |
| [2026-06-26 20:07:44 DP3 TP3] Decode batch, #running-req: 32, #token: 1073472, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.56, #queue-req: 0 | |
| [2026-06-26 20:07:44 DP2 TP2] Decode batch, #running-req: 32, #token: 1071488, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.55, #queue-req: 0 | |
| [2026-06-26 20:07:44 DP0 TP0] Decode batch, #running-req: 32, #token: 1072000, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1118.54, #queue-req: 0 | |
| [2026-06-26 20:07:45 DP1 TP1] Decode batch, #running-req: 32, #token: 1073472, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.53, #queue-req: 0 | |
| [2026-06-26 20:07:45 DP3 TP3] Decode batch, #running-req: 32, #token: 1074816, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.48, #queue-req: 0 | |
| [2026-06-26 20:07:45 DP2 TP2] Decode batch, #running-req: 32, #token: 1072768, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.48, #queue-req: 0 | |
| [2026-06-26 20:07:45 DP0 TP0] Decode batch, #running-req: 32, #token: 1073536, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1117.47, #queue-req: 0 | |
| [2026-06-26 20:07:46 DP1 TP1] Decode batch, #running-req: 32, #token: 1074944, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.68, #queue-req: 0 | |
| [2026-06-26 20:07:46 DP2 TP2] Decode batch, #running-req: 32, #token: 1073792, token usage: 0.60, cuda graph: True, gen throughput (token/s): 1115.83, #queue-req: 0 | |
| [2026-06-26 20:07:46 DP3 TP3] Decode batch, #running-req: 32, #token: 1075968, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.82, #queue-req: 0 | |
| [2026-06-26 20:07:46 DP0 TP0] Decode batch, #running-req: 32, #token: 1074752, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.83, #queue-req: 0 | |
| [2026-06-26 20:07:47 DP1 TP1] Decode batch, #running-req: 32, #token: 1076160, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.94, #queue-req: 0 | |
| [2026-06-26 20:07:47 DP3 TP3] Decode batch, #running-req: 32, #token: 1077376, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.72, #queue-req: 0 | |
| [2026-06-26 20:07:47 DP2 TP2] Decode batch, #running-req: 32, #token: 1075392, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.70, #queue-req: 0 | |
| [2026-06-26 20:07:47 DP0 TP0] Decode batch, #running-req: 32, #token: 1075776, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.72, #queue-req: 0 | |
| [2026-06-26 20:07:48 DP1 TP1] Decode batch, #running-req: 32, #token: 1077568, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.31, #queue-req: 0 | |
| [2026-06-26 20:07:48 DP0 TP0] Decode batch, #running-req: 32, #token: 1077440, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.38, #queue-req: 0 | |
| [2026-06-26 20:07:48 DP3 TP3] Decode batch, #running-req: 32, #token: 1078592, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.34, #queue-req: 0 | |
| [2026-06-26 20:07:48 DP2 TP2] Decode batch, #running-req: 32, #token: 1076288, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.34, #queue-req: 0 | |
| [2026-06-26 20:07:49 DP1 TP1] Decode batch, #running-req: 32, #token: 1078592, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.98, #queue-req: 0 | |
| [2026-06-26 20:07:49 DP0 TP0] Decode batch, #running-req: 32, #token: 1078400, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1118.08, #queue-req: 0 | |
| [2026-06-26 20:07:49 DP2 TP2] Decode batch, #running-req: 32, #token: 1077824, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1118.13, #queue-req: 0 | |
| [2026-06-26 20:07:49 DP3 TP3] Decode batch, #running-req: 32, #token: 1079872, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1118.13, #queue-req: 0 | |
| [2026-06-26 20:07:50] INFO: 127.0.0.1:33560 - "GET /health HTTP/1.1" 200 OK | |
| [2026-06-26 20:07:51 DP1 TP1] Decode batch, #running-req: 32, #token: 1079872, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.06, #queue-req: 0 | |
| [2026-06-26 20:07:51 DP3 TP3] Decode batch, #running-req: 32, #token: 1081152, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.96, #queue-req: 0 | |
| [2026-06-26 20:07:51 DP2 TP2] Decode batch, #running-req: 32, #token: 1079232, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.97, #queue-req: 0 | |
| [2026-06-26 20:07:51 DP0 TP0] Decode batch, #running-req: 32, #token: 1079808, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.96, #queue-req: 0 | |
| [2026-06-26 20:07:52 DP1 TP1] Decode batch, #running-req: 32, #token: 1081472, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.49, #queue-req: 0 | |
| [2026-06-26 20:07:52 DP0 TP0] Decode batch, #running-req: 32, #token: 1081344, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.62, #queue-req: 0 | |
| [2026-06-26 20:07:52 DP3 TP3] Decode batch, #running-req: 32, #token: 1082496, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.61, #queue-req: 0 | |
| [2026-06-26 20:07:52 DP2 TP2] Decode batch, #running-req: 32, #token: 1080320, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.62, #queue-req: 0 | |
| [2026-06-26 20:07:53 DP1 TP1] Decode batch, #running-req: 32, #token: 1082560, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1119.76, #queue-req: 0 | |
| [2026-06-26 20:07:53 DP3 TP3] Decode batch, #running-req: 32, #token: 1083712, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1119.91, #queue-req: 0 | |
| [2026-06-26 20:07:53 DP2 TP2] Decode batch, #running-req: 32, #token: 1081728, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1119.91, #queue-req: 0 | |
| [2026-06-26 20:07:53 DP0 TP0] Decode batch, #running-req: 32, #token: 1082240, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1119.87, #queue-req: 0 | |
| [2026-06-26 20:07:54 DP1 TP1] Decode batch, #running-req: 32, #token: 1083712, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.12, #queue-req: 0 | |
| [2026-06-26 20:07:54 DP0 TP0] Decode batch, #running-req: 32, #token: 1083776, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.94, #queue-req: 0 | |
| [2026-06-26 20:07:54 DP2 TP2] Decode batch, #running-req: 32, #token: 1083008, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.90, #queue-req: 0 | |
| [2026-06-26 20:07:54 DP3 TP3] Decode batch, #running-req: 32, #token: 1085056, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.91, #queue-req: 0 | |
| [2026-06-26 20:07:55 DP1 TP1] Decode batch, #running-req: 32, #token: 1085184, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.55, #queue-req: 0 | |
| [2026-06-26 20:07:55 DP0 TP0] Decode batch, #running-req: 32, #token: 1084992, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.46, #queue-req: 0 | |
| [2026-06-26 20:07:55 DP3 TP3] Decode batch, #running-req: 32, #token: 1086208, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.46, #queue-req: 0 | |
| [2026-06-26 20:07:55 DP2 TP2] Decode batch, #running-req: 32, #token: 1084032, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.46, #queue-req: 0 | |
| [2026-06-26 20:07:56 DP1 TP1] Decode batch, #running-req: 32, #token: 1086400, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.01, #queue-req: 0 | |
| [2026-06-26 20:07:56 DP0 TP0] Decode batch, #running-req: 32, #token: 1086016, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.11, #queue-req: 0 | |
| [2026-06-26 20:07:56 DP2 TP2] Decode batch, #running-req: 32, #token: 1085632, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.11, #queue-req: 0 | |
| [2026-06-26 20:07:56 DP3 TP3] Decode batch, #running-req: 32, #token: 1087616, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.10, #queue-req: 0 | |
| [2026-06-26 20:07:57 DP1 TP1] Decode batch, #running-req: 32, #token: 1087808, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.24, #queue-req: 0 | |
| [2026-06-26 20:07:57 DP2 TP2] Decode batch, #running-req: 32, #token: 1086528, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.18, #queue-req: 0 | |
| [2026-06-26 20:07:57 DP3 TP3] Decode batch, #running-req: 32, #token: 1088832, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.18, #queue-req: 0 | |
| [2026-06-26 20:07:57 DP0 TP0] Decode batch, #running-req: 32, #token: 1087680, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1116.17, #queue-req: 0 | |
| [2026-06-26 20:07:59 DP1 TP1] Decode batch, #running-req: 32, #token: 1088832, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.53, #queue-req: 0 | |
| [2026-06-26 20:07:59 DP0 TP0] Decode batch, #running-req: 32, #token: 1088640, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.57, #queue-req: 0 | |
| [2026-06-26 20:07:59 DP2 TP2] Decode batch, #running-req: 32, #token: 1088064, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.51, #queue-req: 0 | |
| [2026-06-26 20:07:59 DP3 TP3] Decode batch, #running-req: 32, #token: 1090112, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.53, #queue-req: 0 | |
| [2026-06-26 20:08:00 DP1 TP1] Decode batch, #running-req: 32, #token: 1090112, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.80, #queue-req: 0 | |
| [2026-06-26 20:08:00 DP2 TP2] Decode batch, #running-req: 32, #token: 1089472, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.76, #queue-req: 0 | |
| [2026-06-26 20:08:00 DP3 TP3] Decode batch, #running-req: 32, #token: 1091392, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.75, #queue-req: 0 | |
| [2026-06-26 20:08:00 DP0 TP0] Decode batch, #running-req: 32, #token: 1090048, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1115.69, #queue-req: 0 | |
| [2026-06-26 20:08:01 DP1 TP1] Decode batch, #running-req: 32, #token: 1091712, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1114.04, #queue-req: 0 | |
| [2026-06-26 20:08:01 DP3 TP3] Decode batch, #running-req: 32, #token: 1092736, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1114.10, #queue-req: 0 | |
| [2026-06-26 20:08:01 DP2 TP2] Decode batch, #running-req: 32, #token: 1090560, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1114.11, #queue-req: 0 | |
| [2026-06-26 20:08:01 DP0 TP0] Decode batch, #running-req: 32, #token: 1091584, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1114.13, #queue-req: 0 | |
| [2026-06-26 20:08:02 DP1 TP1] Decode batch, #running-req: 32, #token: 1092800, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1117.27, #queue-req: 0 | |
| [2026-06-26 20:08:02 DP3 TP3] Decode batch, #running-req: 32, #token: 1093952, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1117.25, #queue-req: 0 | |
| [2026-06-26 20:08:02 DP2 TP2] Decode batch, #running-req: 32, #token: 1091968, token usage: 0.61, cuda graph: True, gen throughput (token/s): 1117.25, #queue-req: 0 | |
| [2026-06-26 20:08:02 DP0 TP0] Decode batch, #running-req: 32, #token: 1092480, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1117.24, #queue-req: 0 | |
| [2026-06-26 20:08:03 DP1 TP1] Decode batch, #running-req: 32, #token: 1093952, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1115.85, #queue-req: 0 | |
| [2026-06-26 20:08:03 DP3 TP3] Decode batch, #running-req: 32, #token: 1095296, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1115.89, #queue-req: 0 | |
| [2026-06-26 20:08:03 DP2 TP2] Decode batch, #running-req: 32, #token: 1093248, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1115.89, #queue-req: 0 | |
| [2026-06-26 20:08:03 DP0 TP0] Decode batch, #running-req: 32, #token: 1094016, token usage: 0.62, cuda graph: True, gen throughput (token/s): 1115.88, #queue-req: 0 | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47994 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48094 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48118 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48150 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48176 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47766 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47664 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47704 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47754 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47786 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47824 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47860 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47914 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47956 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47998 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48022 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48058 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48210 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48250 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48298 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48324 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48370 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48394 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48446 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48498 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48554 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48594 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48614 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48638 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48670 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48708 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48736 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48764 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47694 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47784 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47812 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47852 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47900 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47940 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47992 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48020 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48052 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48090 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48116 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48140 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48174 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48204 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48240 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48286 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48314 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48354 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48388 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48440 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48490 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48540 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48584 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48610 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48636 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48660 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48706 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48724 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48750 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47732 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47668 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47718 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47772 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47794 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47832 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47870 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47918 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47964 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48006 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48038 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48072 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48100 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48128 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48160 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48186 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48214 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48266 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48300 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48340 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48376 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48410 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48462 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48514 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48570 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48596 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48626 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48652 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48686 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48714 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48738 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48776 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47648 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47682 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47742 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47774 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47806 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47846 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47886 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47932 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:47980 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48008 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48044 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48086 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48108 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48138 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48164 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48198 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48228 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48280 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48310 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48350 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48382 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48424 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48476 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48524 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48576 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48598 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48630 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48654 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48720 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:04] INFO: 127.0.0.1:48780 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:08:08 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.05, #running-req: 0, #queue-req: 2, #pending-token: 0, cuda graph: False, input throughput (token/s): 660.52 | |
| [2026-06-26 20:08:11 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.05, #running-req: 0, #queue-req: 12, #pending-token: 0, cuda graph: False, input throughput (token/s): 632.28 | |
| [2026-06-26 20:08:11 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 22912, token usage: 0.05, #running-req: 0, #queue-req: 12, #pending-token: 0, cuda graph: False, input throughput (token/s): 281.03 | |
| [2026-06-26 20:08:11 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.05, #running-req: 0, #queue-req: 12, #pending-token: 0, cuda graph: False, input throughput (token/s): 633.25 | |
| [2026-06-26 20:08:11 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32064, token usage: 0.07, #running-req: 1, #queue-req: 11, #pending-token: 82678, cuda graph: False, input throughput (token/s): 3086.04 | |
| [2026-06-26 20:08:13 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.09, #running-req: 2, #queue-req: 22, #pending-token: 453654, cuda graph: False, input throughput (token/s): 16411.05 | |
| [2026-06-26 20:08:13 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.07, #running-req: 1, #queue-req: 23, #pending-token: 495110, cuda graph: False, input throughput (token/s): 16359.82 | |
| [2026-06-26 20:08:13 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 42112, #cached-token: 0, token usage: 0.07, #running-req: 1, #queue-req: 23, #pending-token: 495113, cuda graph: False, input throughput (token/s): 16689.49 | |
| [2026-06-26 20:08:13 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32384, token usage: 0.07, #running-req: 1, #queue-req: 23, #pending-token: 496240, cuda graph: False, input throughput (token/s): 3622.19 | |
| [2026-06-26 20:08:16 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32064, token usage: 0.09, #running-req: 2, #queue-req: 28, #pending-token: 950151, cuda graph: False, input throughput (token/s): 3492.08 | |
| [2026-06-26 20:08:16 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 32000, token usage: 0.09, #running-req: 2, #queue-req: 28, #pending-token: 948692, cuda graph: False, input throughput (token/s): 3516.29 | |
| [2026-06-26 20:08:16 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.09, #running-req: 2, #queue-req: 28, #pending-token: 950525, cuda graph: False, input throughput (token/s): 15649.44 | |
| [2026-06-26 20:08:16 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32064, token usage: 0.12, #running-req: 3, #queue-req: 27, #pending-token: 907975, cuda graph: False, input throughput (token/s): 3489.36 | |
| [2026-06-26 20:08:19 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.12, #running-req: 3, #queue-req: 27, #pending-token: 1157555, cuda graph: False, input throughput (token/s): 14285.91 | |
| [2026-06-26 20:08:19 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.14, #running-req: 4, #queue-req: 26, #pending-token: 1114524, cuda graph: False, input throughput (token/s): 14241.61 | |
| [2026-06-26 20:08:19 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 18368, #cached-token: 22912, token usage: 0.12, #running-req: 3, #queue-req: 27, #pending-token: 1156251, cuda graph: False, input throughput (token/s): 6332.42 | |
| [2026-06-26 20:08:19 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 18368, #cached-token: 22912, token usage: 0.12, #running-req: 3, #queue-req: 27, #pending-token: 1155216, cuda graph: False, input throughput (token/s): 6332.22 | |
| [2026-06-26 20:08:21 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 18368, #cached-token: 22912, token usage: 0.16, #running-req: 5, #queue-req: 25, #pending-token: 1073256, cuda graph: False, input throughput (token/s): 6787.08 | |
| [2026-06-26 20:08:21 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 42304, #cached-token: 0, token usage: 0.14, #running-req: 4, #queue-req: 26, #pending-token: 1115282, cuda graph: False, input throughput (token/s): 15630.67 | |
| [2026-06-26 20:08:21 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 22912, token usage: 0.14, #running-req: 4, #queue-req: 26, #pending-token: 1114062, cuda graph: False, input throughput (token/s): 6755.88 | |
| [2026-06-26 20:08:21 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.14, #running-req: 4, #queue-req: 26, #pending-token: 1115031, cuda graph: False, input throughput (token/s): 15235.00 | |
| [2026-06-26 20:08:25 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.16, #running-req: 5, #queue-req: 25, #pending-token: 1072777, cuda graph: False, input throughput (token/s): 13134.98 | |
| [2026-06-26 20:08:25 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32000, token usage: 0.19, #running-req: 6, #queue-req: 24, #pending-token: 1032106, cuda graph: False, input throughput (token/s): 2904.66 | |
| [2026-06-26 20:08:25 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.16, #running-req: 5, #queue-req: 25, #pending-token: 1073864, cuda graph: False, input throughput (token/s): 13094.37 | |
| [2026-06-26 20:08:25 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32064, token usage: 0.16, #running-req: 5, #queue-req: 25, #pending-token: 1074075, cuda graph: False, input throughput (token/s): 2904.58 | |
| [2026-06-26 20:08:28 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.21, #running-req: 7, #queue-req: 23, #pending-token: 990758, cuda graph: False, input throughput (token/s): 13711.47 | |
| [2026-06-26 20:08:28 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.19, #running-req: 6, #queue-req: 24, #pending-token: 1031545, cuda graph: False, input throughput (token/s): 13668.73 | |
| [2026-06-26 20:08:28 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.19, #running-req: 6, #queue-req: 24, #pending-token: 1032935, cuda graph: False, input throughput (token/s): 13618.03 | |
| [2026-06-26 20:08:28 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.19, #running-req: 6, #queue-req: 24, #pending-token: 1032748, cuda graph: False, input throughput (token/s): 13617.31 | |
| [2026-06-26 20:08:31 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.21, #running-req: 7, #queue-req: 23, #pending-token: 991540, cuda graph: False, input throughput (token/s): 12711.85 | |
| [2026-06-26 20:08:31 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.21, #running-req: 7, #queue-req: 23, #pending-token: 991520, cuda graph: False, input throughput (token/s): 12672.86 | |
| [2026-06-26 20:08:31 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 22912, token usage: 0.23, #running-req: 8, #queue-req: 22, #pending-token: 949580, cuda graph: False, input throughput (token/s): 5615.14 | |
| [2026-06-26 20:08:31 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 18496, #cached-token: 22912, token usage: 0.21, #running-req: 7, #queue-req: 23, #pending-token: 990142, cuda graph: False, input throughput (token/s): 5674.10 | |
| [2026-06-26 20:08:34 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.26, #running-req: 9, #queue-req: 21, #pending-token: 908317, cuda graph: False, input throughput (token/s): 15049.51 | |
| [2026-06-26 20:08:34 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.23, #running-req: 8, #queue-req: 22, #pending-token: 948946, cuda graph: False, input throughput (token/s): 15026.38 | |
| [2026-06-26 20:08:34 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.23, #running-req: 8, #queue-req: 22, #pending-token: 950263, cuda graph: False, input throughput (token/s): 15043.62 | |
| [2026-06-26 20:08:34 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.23, #running-req: 8, #queue-req: 22, #pending-token: 950352, cuda graph: False, input throughput (token/s): 15007.25 | |
| [2026-06-26 20:08:36 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.26, #running-req: 9, #queue-req: 21, #pending-token: 908962, cuda graph: False, input throughput (token/s): 14676.43 | |
| [2026-06-26 20:08:36 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 18560, #cached-token: 22912, token usage: 0.26, #running-req: 9, #queue-req: 21, #pending-token: 908836, cuda graph: False, input throughput (token/s): 6572.63 | |
| [2026-06-26 20:08:36 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9088, #cached-token: 32064, token usage: 0.28, #running-req: 10, #queue-req: 20, #pending-token: 867192, cuda graph: False, input throughput (token/s): 3217.27 | |
| [2026-06-26 20:08:36 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32064, token usage: 0.26, #running-req: 9, #queue-req: 21, #pending-token: 907676, cuda graph: False, input throughput (token/s): 3262.54 | |
| [2026-06-26 20:08:39 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.28, #running-req: 10, #queue-req: 20, #pending-token: 867624, cuda graph: False, input throughput (token/s): 14697.78 | |
| [2026-06-26 20:08:39 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.28, #running-req: 10, #queue-req: 20, #pending-token: 867707, cuda graph: False, input throughput (token/s): 14720.44 | |
| [2026-06-26 20:08:39 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.30, #running-req: 11, #queue-req: 19, #pending-token: 826009, cuda graph: False, input throughput (token/s): 14698.06 | |
| [2026-06-26 20:08:39 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 32256, token usage: 0.28, #running-req: 10, #queue-req: 20, #pending-token: 866170, cuda graph: False, input throughput (token/s): 3309.21 | |
| [2026-06-26 20:08:42 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.30, #running-req: 11, #queue-req: 19, #pending-token: 826297, cuda graph: False, input throughput (token/s): 15897.64 | |
| [2026-06-26 20:08:42 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41728, #cached-token: 0, token usage: 0.30, #running-req: 11, #queue-req: 19, #pending-token: 826029, cuda graph: False, input throughput (token/s): 16021.38 | |
| [2026-06-26 20:08:42 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 32192, token usage: 0.33, #running-req: 12, #queue-req: 18, #pending-token: 784583, cuda graph: False, input throughput (token/s): 3550.96 | |
| [2026-06-26 20:08:42 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32000, token usage: 0.30, #running-req: 11, #queue-req: 19, #pending-token: 825005, cuda graph: False, input throughput (token/s): 3526.45 | |
| [2026-06-26 20:08:45 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.35, #running-req: 13, #queue-req: 17, #pending-token: 743410, cuda graph: False, input throughput (token/s): 14612.43 | |
| [2026-06-26 20:08:45 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32000, token usage: 0.33, #running-req: 12, #queue-req: 18, #pending-token: 785147, cuda graph: False, input throughput (token/s): 3229.81 | |
| [2026-06-26 20:08:45 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 22912, token usage: 0.33, #running-req: 12, #queue-req: 18, #pending-token: 784873, cuda graph: False, input throughput (token/s): 6468.48 | |
| [2026-06-26 20:08:45 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.33, #running-req: 12, #queue-req: 18, #pending-token: 783828, cuda graph: False, input throughput (token/s): 14612.28 | |
| [2026-06-26 20:08:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.35, #running-req: 13, #queue-req: 17, #pending-token: 742586, cuda graph: False, input throughput (token/s): 15582.26 | |
| [2026-06-26 20:08:47 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32064, token usage: 0.35, #running-req: 13, #queue-req: 17, #pending-token: 743661, cuda graph: False, input throughput (token/s): 3454.49 | |
| [2026-06-26 20:08:47 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.37, #running-req: 14, #queue-req: 16, #pending-token: 702004, cuda graph: False, input throughput (token/s): 15612.46 | |
| [2026-06-26 20:08:47 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 22912, token usage: 0.35, #running-req: 13, #queue-req: 17, #pending-token: 743966, cuda graph: False, input throughput (token/s): 6901.12 | |
| [2026-06-26 20:08:50 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.40, #running-req: 15, #queue-req: 15, #pending-token: 660807, cuda graph: False, input throughput (token/s): 13078.30 | |
| [2026-06-26 20:08:50 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 31936, token usage: 0.37, #running-req: 14, #queue-req: 16, #pending-token: 701458, cuda graph: False, input throughput (token/s): 2921.54 | |
| [2026-06-26 20:08:50 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.37, #running-req: 14, #queue-req: 16, #pending-token: 702728, cuda graph: False, input throughput (token/s): 13098.79 | |
| [2026-06-26 20:08:50 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32000, token usage: 0.37, #running-req: 14, #queue-req: 16, #pending-token: 702503, cuda graph: False, input throughput (token/s): 2921.57 | |
| [2026-06-26 20:08:54 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41728, #cached-token: 0, token usage: 0.40, #running-req: 15, #queue-req: 15, #pending-token: 660836, cuda graph: False, input throughput (token/s): 12067.66 | |
| [2026-06-26 20:08:54 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.40, #running-req: 15, #queue-req: 15, #pending-token: 661445, cuda graph: False, input throughput (token/s): 11956.55 | |
| [2026-06-26 20:08:54 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.42, #running-req: 16, #queue-req: 14, #pending-token: 619535, cuda graph: False, input throughput (token/s): 11936.81 | |
| [2026-06-26 20:08:54 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.40, #running-req: 15, #queue-req: 15, #pending-token: 660161, cuda graph: False, input throughput (token/s): 11955.55 | |
| [2026-06-26 20:08:57 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41920, #cached-token: 0, token usage: 0.44, #running-req: 17, #queue-req: 13, #pending-token: 577637, cuda graph: False, input throughput (token/s): 12236.44 | |
| [2026-06-26 20:08:57 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.42, #running-req: 16, #queue-req: 14, #pending-token: 618864, cuda graph: False, input throughput (token/s): 12068.17 | |
| [2026-06-26 20:08:57 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41536, #cached-token: 0, token usage: 0.42, #running-req: 16, #queue-req: 14, #pending-token: 619315, cuda graph: False, input throughput (token/s): 12123.37 | |
| [2026-06-26 20:08:57 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.42, #running-req: 16, #queue-req: 14, #pending-token: 620201, cuda graph: False, input throughput (token/s): 12048.58 | |
| [2026-06-26 20:09:00 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.47, #running-req: 18, #queue-req: 12, #pending-token: 536529, cuda graph: False, input throughput (token/s): 14238.84 | |
| [2026-06-26 20:09:00 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.44, #running-req: 17, #queue-req: 13, #pending-token: 578937, cuda graph: False, input throughput (token/s): 14283.59 | |
| [2026-06-26 20:09:00 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.44, #running-req: 17, #queue-req: 13, #pending-token: 577851, cuda graph: False, input throughput (token/s): 14333.70 | |
| [2026-06-26 20:09:00 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.44, #running-req: 17, #queue-req: 13, #pending-token: 577560, cuda graph: False, input throughput (token/s): 14287.62 | |
| [2026-06-26 20:09:03 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32000, token usage: 0.47, #running-req: 18, #queue-req: 12, #pending-token: 537734, cuda graph: False, input throughput (token/s): 3792.49 | |
| [2026-06-26 20:09:03 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.47, #running-req: 18, #queue-req: 12, #pending-token: 536627, cuda graph: False, input throughput (token/s): 17010.06 | |
| [2026-06-26 20:09:03 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.47, #running-req: 18, #queue-req: 12, #pending-token: 536399, cuda graph: False, input throughput (token/s): 16986.07 | |
| [2026-06-26 20:09:03 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 32128, token usage: 0.49, #running-req: 19, #queue-req: 11, #pending-token: 495094, cuda graph: False, input throughput (token/s): 3844.07 | |
| [2026-06-26 20:09:06 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.49, #running-req: 19, #queue-req: 11, #pending-token: 496473, cuda graph: False, input throughput (token/s): 13841.83 | |
| [2026-06-26 20:09:06 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32192, token usage: 0.51, #running-req: 20, #queue-req: 10, #pending-token: 453707, cuda graph: False, input throughput (token/s): 3090.76 | |
| [2026-06-26 20:09:06 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32000, token usage: 0.49, #running-req: 19, #queue-req: 11, #pending-token: 495234, cuda graph: False, input throughput (token/s): 3090.11 | |
| [2026-06-26 20:09:06 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32064, token usage: 0.49, #running-req: 19, #queue-req: 11, #pending-token: 495394, cuda graph: False, input throughput (token/s): 3089.94 | |
| [2026-06-26 20:09:09 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.53, #running-req: 21, #queue-req: 9, #pending-token: 412571, cuda graph: False, input throughput (token/s): 12011.87 | |
| [2026-06-26 20:09:09 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.51, #running-req: 20, #queue-req: 10, #pending-token: 454064, cuda graph: False, input throughput (token/s): 12030.44 | |
| [2026-06-26 20:09:09 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.51, #running-req: 20, #queue-req: 10, #pending-token: 455043, cuda graph: False, input throughput (token/s): 12103.42 | |
| [2026-06-26 20:09:09 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41984, #cached-token: 0, token usage: 0.51, #running-req: 20, #queue-req: 10, #pending-token: 453410, cuda graph: False, input throughput (token/s): 12253.22 | |
| [2026-06-26 20:09:12 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.54, #running-req: 21, #queue-req: 9, #pending-token: 413877, cuda graph: False, input throughput (token/s): 12732.63 | |
| [2026-06-26 20:09:12 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.54, #running-req: 21, #queue-req: 9, #pending-token: 412201, cuda graph: False, input throughput (token/s): 12732.64 | |
| [2026-06-26 20:09:12 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.56, #running-req: 22, #queue-req: 8, #pending-token: 371409, cuda graph: False, input throughput (token/s): 12729.18 | |
| [2026-06-26 20:09:12 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.53, #running-req: 21, #queue-req: 9, #pending-token: 412821, cuda graph: False, input throughput (token/s): 12746.92 | |
| [2026-06-26 20:09:16 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.56, #running-req: 22, #queue-req: 8, #pending-token: 372770, cuda graph: False, input throughput (token/s): 11998.06 | |
| [2026-06-26 20:09:16 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.56, #running-req: 22, #queue-req: 8, #pending-token: 371665, cuda graph: False, input throughput (token/s): 12020.43 | |
| [2026-06-26 20:09:16 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.56, #running-req: 22, #queue-req: 8, #pending-token: 370989, cuda graph: False, input throughput (token/s): 12013.75 | |
| [2026-06-26 20:09:16 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.58, #running-req: 23, #queue-req: 7, #pending-token: 330180, cuda graph: False, input throughput (token/s): 11993.80 | |
| [2026-06-26 20:09:19 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.58, #running-req: 23, #queue-req: 7, #pending-token: 331449, cuda graph: False, input throughput (token/s): 12095.26 | |
| [2026-06-26 20:09:19 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.60, #running-req: 24, #queue-req: 6, #pending-token: 288897, cuda graph: False, input throughput (token/s): 12138.82 | |
| [2026-06-26 20:09:19 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.58, #running-req: 23, #queue-req: 7, #pending-token: 330232, cuda graph: False, input throughput (token/s): 12132.18 | |
| [2026-06-26 20:09:19 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.58, #running-req: 23, #queue-req: 7, #pending-token: 329706, cuda graph: False, input throughput (token/s): 12097.16 | |
| [2026-06-26 20:09:23 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41088, #cached-token: 0, token usage: 0.63, #running-req: 25, #queue-req: 5, #pending-token: 247811, cuda graph: False, input throughput (token/s): 12217.12 | |
| [2026-06-26 20:09:23 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.61, #running-req: 24, #queue-req: 6, #pending-token: 290189, cuda graph: False, input throughput (token/s): 12273.64 | |
| [2026-06-26 20:09:23 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.61, #running-req: 24, #queue-req: 6, #pending-token: 288504, cuda graph: False, input throughput (token/s): 12256.53 | |
| [2026-06-26 20:09:23 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41344, #cached-token: 0, token usage: 0.60, #running-req: 24, #queue-req: 6, #pending-token: 288945, cuda graph: False, input throughput (token/s): 12292.82 | |
| [2026-06-26 20:09:26 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.63, #running-req: 25, #queue-req: 5, #pending-token: 247364, cuda graph: False, input throughput (token/s): 12037.93 | |
| [2026-06-26 20:09:26 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41792, #cached-token: 0, token usage: 0.63, #running-req: 25, #queue-req: 5, #pending-token: 248426, cuda graph: False, input throughput (token/s): 12225.16 | |
| [2026-06-26 20:09:26 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.65, #running-req: 26, #queue-req: 4, #pending-token: 206607, cuda graph: False, input throughput (token/s): 12056.42 | |
| [2026-06-26 20:09:26 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.63, #running-req: 25, #queue-req: 5, #pending-token: 247764, cuda graph: False, input throughput (token/s): 12056.72 | |
| [2026-06-26 20:09:29 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.65, #running-req: 26, #queue-req: 4, #pending-token: 205939, cuda graph: False, input throughput (token/s): 13250.64 | |
| [2026-06-26 20:09:29 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.65, #running-req: 26, #queue-req: 4, #pending-token: 207154, cuda graph: False, input throughput (token/s): 13176.63 | |
| [2026-06-26 20:09:29 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.65, #running-req: 26, #queue-req: 4, #pending-token: 206632, cuda graph: False, input throughput (token/s): 13135.57 | |
| [2026-06-26 20:09:29 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41536, #cached-token: 0, token usage: 0.67, #running-req: 27, #queue-req: 3, #pending-token: 165095, cuda graph: False, input throughput (token/s): 13255.55 | |
| [2026-06-26 20:09:32 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.68, #running-req: 27, #queue-req: 3, #pending-token: 165931, cuda graph: False, input throughput (token/s): 13559.25 | |
| [2026-06-26 20:09:32 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 32000, token usage: 0.68, #running-req: 27, #queue-req: 3, #pending-token: 164754, cuda graph: False, input throughput (token/s): 3024.15 | |
| [2026-06-26 20:09:32 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.70, #running-req: 28, #queue-req: 2, #pending-token: 123816, cuda graph: False, input throughput (token/s): 13561.90 | |
| [2026-06-26 20:09:32 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.67, #running-req: 27, #queue-req: 3, #pending-token: 165277, cuda graph: False, input throughput (token/s): 13601.17 | |
| [2026-06-26 20:09:35 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.70, #running-req: 28, #queue-req: 2, #pending-token: 124769, cuda graph: False, input throughput (token/s): 12649.82 | |
| [2026-06-26 20:09:35 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.70, #running-req: 28, #queue-req: 2, #pending-token: 123654, cuda graph: False, input throughput (token/s): 12630.34 | |
| [2026-06-26 20:09:35 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41280, #cached-token: 0, token usage: 0.72, #running-req: 29, #queue-req: 1, #pending-token: 82550, cuda graph: False, input throughput (token/s): 12669.24 | |
| [2026-06-26 20:09:35 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 32000, token usage: 0.70, #running-req: 28, #queue-req: 2, #pending-token: 124143, cuda graph: False, input throughput (token/s): 2808.86 | |
| [2026-06-26 20:09:39 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.74, #running-req: 30, #queue-req: 0, #pending-token: 41185, cuda graph: False, input throughput (token/s): 12219.56 | |
| [2026-06-26 20:09:39 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.72, #running-req: 29, #queue-req: 1, #pending-token: 82509, cuda graph: False, input throughput (token/s): 12143.46 | |
| [2026-06-26 20:09:39 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.72, #running-req: 29, #queue-req: 1, #pending-token: 82780, cuda graph: False, input throughput (token/s): 12217.57 | |
| [2026-06-26 20:09:39 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.72, #running-req: 29, #queue-req: 1, #pending-token: 83312, cuda graph: False, input throughput (token/s): 12235.01 | |
| [2026-06-26 20:09:42 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.74, #running-req: 30, #queue-req: 0, #pending-token: 41431, cuda graph: False, input throughput (token/s): 13369.78 | |
| [2026-06-26 20:09:42 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.75, #running-req: 30, #queue-req: 0, #pending-token: 42117, cuda graph: False, input throughput (token/s): 13308.36 | |
| [2026-06-26 20:09:42 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41408, #cached-token: 0, token usage: 0.75, #running-req: 30, #queue-req: 0, #pending-token: 41133, cuda graph: False, input throughput (token/s): 13364.65 | |
| [2026-06-26 20:09:42 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 41216, #cached-token: 0, token usage: 0.74, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 13005.31 | |
| [2026-06-26 20:09:43 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 41152, #cached-token: 0, token usage: 0.75, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 40212.22 | |
| [2026-06-26 20:09:43 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 41472, #cached-token: 0, token usage: 0.74, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 40495.29 | |
| [2026-06-26 20:09:43 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 42176, #cached-token: 0, token usage: 0.75, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 41135.58 | |
| [2026-06-26 20:09:43 DP1 TP1] Decode batch, #running-req: 32, #token: 1324480, token usage: 0.75, cuda graph: True, gen throughput (token/s): 12.78, #queue-req: 0 | |
| [2026-06-26 20:09:43 DP0 TP0] Decode batch, #running-req: 32, #token: 1323648, token usage: 0.75, cuda graph: True, gen throughput (token/s): 12.79, #queue-req: 0 | |
| [2026-06-26 20:09:43 DP2 TP2] Decode batch, #running-req: 32, #token: 1322368, token usage: 0.74, cuda graph: True, gen throughput (token/s): 12.79, #queue-req: 0 | |
| [2026-06-26 20:09:43 DP3 TP3] Decode batch, #running-req: 32, #token: 1321728, token usage: 0.74, cuda graph: True, gen throughput (token/s): 12.79, #queue-req: 0 | |
| [2026-06-26 20:09:44 DP2 TP2] Decode batch, #running-req: 32, #token: 1323584, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1155.87, #queue-req: 0 | |
| [2026-06-26 20:09:44 DP1 TP1] Decode batch, #running-req: 32, #token: 1325696, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1155.83, #queue-req: 0 | |
| [2026-06-26 20:09:44 DP0 TP0] Decode batch, #running-req: 32, #token: 1324736, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1155.84, #queue-req: 0 | |
| [2026-06-26 20:09:44 DP3 TP3] Decode batch, #running-req: 32, #token: 1323264, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1154.48, #queue-req: 0 | |
| [2026-06-26 20:09:45 DP0 TP0] Decode batch, #running-req: 32, #token: 1326080, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1133.76, #queue-req: 0 | |
| [2026-06-26 20:09:45 DP1 TP1] Decode batch, #running-req: 32, #token: 1327168, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1133.75, #queue-req: 0 | |
| [2026-06-26 20:09:45 DP2 TP2] Decode batch, #running-req: 32, #token: 1325056, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1133.73, #queue-req: 0 | |
| [2026-06-26 20:09:46 DP3 TP3] Decode batch, #running-req: 32, #token: 1324096, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1133.63, #queue-req: 0 | |
| [2026-06-26 20:09:47 DP1 TP1] Decode batch, #running-req: 32, #token: 1328128, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1128.02, #queue-req: 0 | |
| [2026-06-26 20:09:47 DP0 TP0] Decode batch, #running-req: 32, #token: 1327488, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1128.01, #queue-req: 0 | |
| [2026-06-26 20:09:47 DP2 TP2] Decode batch, #running-req: 32, #token: 1326208, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1128.01, #queue-req: 0 | |
| [2026-06-26 20:09:47 DP3 TP3] Decode batch, #running-req: 32, #token: 1325696, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1127.94, #queue-req: 0 | |
| [2026-06-26 20:09:48 DP0 TP0] Decode batch, #running-req: 32, #token: 1328448, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1120.48, #queue-req: 0 | |
| [2026-06-26 20:09:48 DP1 TP1] Decode batch, #running-req: 32, #token: 1329600, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1120.48, #queue-req: 0 | |
| [2026-06-26 20:09:48 DP2 TP2] Decode batch, #running-req: 32, #token: 1327552, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1120.48, #queue-req: 0 | |
| [2026-06-26 20:09:48 DP3 TP3] Decode batch, #running-req: 32, #token: 1327104, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1120.23, #queue-req: 0 | |
| [2026-06-26 20:09:49 DP2 TP2] Decode batch, #running-req: 32, #token: 1328960, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.39, #queue-req: 0 | |
| [2026-06-26 20:09:49 DP1 TP1] Decode batch, #running-req: 32, #token: 1331008, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.37, #queue-req: 0 | |
| [2026-06-26 20:09:49 DP0 TP0] Decode batch, #running-req: 32, #token: 1330048, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.36, #queue-req: 0 | |
| [2026-06-26 20:09:49 DP3 TP3] Decode batch, #running-req: 32, #token: 1328128, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.40, #queue-req: 0 | |
| [2026-06-26 20:09:50 DP0 TP0] Decode batch, #running-req: 32, #token: 1331264, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.91, #queue-req: 0 | |
| [2026-06-26 20:09:50 DP1 TP1] Decode batch, #running-req: 32, #token: 1332032, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.91, #queue-req: 0 | |
| [2026-06-26 20:09:50 DP2 TP2] Decode batch, #running-req: 32, #token: 1329856, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.90, #queue-req: 0 | |
| [2026-06-26 20:09:50 DP3 TP3] Decode batch, #running-req: 32, #token: 1329728, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1116.82, #queue-req: 0 | |
| [2026-06-26 20:09:51 DP0 TP0] Decode batch, #running-req: 32, #token: 1332352, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.89, #queue-req: 0 | |
| [2026-06-26 20:09:51 DP2 TP2] Decode batch, #running-req: 32, #token: 1331392, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.88, #queue-req: 0 | |
| [2026-06-26 20:09:51 DP1 TP1] Decode batch, #running-req: 32, #token: 1333440, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.60, #queue-req: 0 | |
| [2026-06-26 20:09:51 DP3 TP3] Decode batch, #running-req: 32, #token: 1330752, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.99, #queue-req: 0 | |
| [2026-06-26 20:09:52 DP0 TP0] Decode batch, #running-req: 32, #token: 1333888, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.27, #queue-req: 0 | |
| [2026-06-26 20:09:52 DP1 TP1] Decode batch, #running-req: 32, #token: 1334720, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.56, #queue-req: 0 | |
| [2026-06-26 20:09:52 DP2 TP2] Decode batch, #running-req: 32, #token: 1332608, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.28, #queue-req: 0 | |
| [2026-06-26 20:09:52 DP3 TP3] Decode batch, #running-req: 32, #token: 1331968, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.23, #queue-req: 0 | |
| [2026-06-26 20:09:54 DP1 TP1] Decode batch, #running-req: 32, #token: 1335936, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.69, #queue-req: 0 | |
| [2026-06-26 20:09:54 DP0 TP0] Decode batch, #running-req: 32, #token: 1334976, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.68, #queue-req: 0 | |
| [2026-06-26 20:09:54 DP2 TP2] Decode batch, #running-req: 32, #token: 1333824, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.67, #queue-req: 0 | |
| [2026-06-26 20:09:54 DP3 TP3] Decode batch, #running-req: 32, #token: 1333504, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1113.73, #queue-req: 0 | |
| [2026-06-26 20:09:55 DP2 TP2] Decode batch, #running-req: 32, #token: 1335296, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.26, #queue-req: 0 | |
| [2026-06-26 20:09:55 DP1 TP1] Decode batch, #running-req: 32, #token: 1337408, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.20, #queue-req: 0 | |
| [2026-06-26 20:09:55 DP0 TP0] Decode batch, #running-req: 32, #token: 1336320, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.21, #queue-req: 0 | |
| [2026-06-26 20:09:55 DP3 TP3] Decode batch, #running-req: 32, #token: 1334336, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1114.15, #queue-req: 0 | |
| [2026-06-26 20:09:56 DP1 TP1] Decode batch, #running-req: 32, #token: 1338368, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.75, #queue-req: 0 | |
| [2026-06-26 20:09:56 DP0 TP0] Decode batch, #running-req: 32, #token: 1337728, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.74, #queue-req: 0 | |
| [2026-06-26 20:09:56 DP2 TP2] Decode batch, #running-req: 32, #token: 1336448, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.71, #queue-req: 0 | |
| [2026-06-26 20:09:56 DP3 TP3] Decode batch, #running-req: 32, #token: 1335936, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.69, #queue-req: 0 | |
| [2026-06-26 20:09:57 DP0 TP0] Decode batch, #running-req: 32, #token: 1338688, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.40, #queue-req: 0 | |
| [2026-06-26 20:09:57 DP1 TP1] Decode batch, #running-req: 32, #token: 1339840, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.38, #queue-req: 0 | |
| [2026-06-26 20:09:57 DP2 TP2] Decode batch, #running-req: 32, #token: 1337792, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.40, #queue-req: 0 | |
| [2026-06-26 20:09:57 DP3 TP3] Decode batch, #running-req: 32, #token: 1337344, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1112.38, #queue-req: 0 | |
| [2026-06-26 20:09:58 DP0 TP0] Decode batch, #running-req: 32, #token: 1340288, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1110.91, #queue-req: 0 | |
| [2026-06-26 20:09:58 DP1 TP1] Decode batch, #running-req: 32, #token: 1341248, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.92, #queue-req: 0 | |
| [2026-06-26 20:09:58 DP2 TP2] Decode batch, #running-req: 32, #token: 1339200, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1110.90, #queue-req: 0 | |
| [2026-06-26 20:09:58 DP3 TP3] Decode batch, #running-req: 32, #token: 1338368, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1110.90, #queue-req: 0 | |
| [2026-06-26 20:09:59 DP1 TP1] Decode batch, #running-req: 32, #token: 1342272, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.37, #queue-req: 0 | |
| [2026-06-26 20:09:59 DP0 TP0] Decode batch, #running-req: 32, #token: 1341504, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.37, #queue-req: 0 | |
| [2026-06-26 20:09:59 DP2 TP2] Decode batch, #running-req: 32, #token: 1340096, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1110.37, #queue-req: 0 | |
| [2026-06-26 20:09:59 DP3 TP3] Decode batch, #running-req: 32, #token: 1339968, token usage: 0.75, cuda graph: True, gen throughput (token/s): 1110.32, #queue-req: 0 | |
| [2026-06-26 20:10:00 DP1 TP1] Decode batch, #running-req: 32, #token: 1343680, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.40, #queue-req: 0 | |
| [2026-06-26 20:10:00 DP0 TP0] Decode batch, #running-req: 32, #token: 1342592, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.39, #queue-req: 0 | |
| [2026-06-26 20:10:00 DP2 TP2] Decode batch, #running-req: 32, #token: 1341632, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.40, #queue-req: 0 | |
| [2026-06-26 20:10:00 DP3 TP3] Decode batch, #running-req: 32, #token: 1340992, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.51, #queue-req: 0 | |
| [2026-06-26 20:10:02 DP0 TP0] Decode batch, #running-req: 32, #token: 1344128, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1112.69, #queue-req: 0 | |
| [2026-06-26 20:10:02 DP1 TP1] Decode batch, #running-req: 32, #token: 1344960, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1112.69, #queue-req: 0 | |
| [2026-06-26 20:10:02 DP2 TP2] Decode batch, #running-req: 32, #token: 1342848, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1112.70, #queue-req: 0 | |
| [2026-06-26 20:10:02 DP3 TP3] Decode batch, #running-req: 32, #token: 1342208, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1112.64, #queue-req: 0 | |
| [2026-06-26 20:10:03 DP0 TP0] Decode batch, #running-req: 32, #token: 1345216, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.80, #queue-req: 0 | |
| [2026-06-26 20:10:03 DP1 TP1] Decode batch, #running-req: 32, #token: 1346176, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.79, #queue-req: 0 | |
| [2026-06-26 20:10:03 DP2 TP2] Decode batch, #running-req: 32, #token: 1344064, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.79, #queue-req: 0 | |
| [2026-06-26 20:10:03 DP3 TP3] Decode batch, #running-req: 32, #token: 1343744, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.75, #queue-req: 0 | |
| [2026-06-26 20:10:04 DP1 TP1] Decode batch, #running-req: 32, #token: 1347648, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.62, #queue-req: 0 | |
| [2026-06-26 20:10:04 DP0 TP0] Decode batch, #running-req: 32, #token: 1346560, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.61, #queue-req: 0 | |
| [2026-06-26 20:10:04 DP2 TP2] Decode batch, #running-req: 32, #token: 1345536, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.61, #queue-req: 0 | |
| [2026-06-26 20:10:04 DP3 TP3] Decode batch, #running-req: 32, #token: 1344576, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.59, #queue-req: 0 | |
| [2026-06-26 20:10:05 DP1 TP1] Decode batch, #running-req: 32, #token: 1348608, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.54, #queue-req: 0 | |
| [2026-06-26 20:10:05 DP0 TP0] Decode batch, #running-req: 32, #token: 1347968, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.54, #queue-req: 0 | |
| [2026-06-26 20:10:05 DP2 TP2] Decode batch, #running-req: 32, #token: 1346688, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.55, #queue-req: 0 | |
| [2026-06-26 20:10:05 DP3 TP3] Decode batch, #running-req: 32, #token: 1346176, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.79, #queue-req: 0 | |
| [2026-06-26 20:10:06 DP1 TP1] Decode batch, #running-req: 32, #token: 1350080, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.98, #queue-req: 0 | |
| [2026-06-26 20:10:06 DP0 TP0] Decode batch, #running-req: 32, #token: 1348928, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.98, #queue-req: 0 | |
| [2026-06-26 20:10:06 DP2 TP2] Decode batch, #running-req: 32, #token: 1348032, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.98, #queue-req: 0 | |
| [2026-06-26 20:10:06 DP3 TP3] Decode batch, #running-req: 32, #token: 1347584, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.92, #queue-req: 0 | |
| [2026-06-26 20:10:07 DP2 TP2] Decode batch, #running-req: 32, #token: 1349440, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.65, #queue-req: 0 | |
| [2026-06-26 20:10:07 DP1 TP1] Decode batch, #running-req: 32, #token: 1351488, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.64, #queue-req: 0 | |
| [2026-06-26 20:10:07 DP0 TP0] Decode batch, #running-req: 32, #token: 1350528, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.64, #queue-req: 0 | |
| [2026-06-26 20:10:07 DP3 TP3] Decode batch, #running-req: 32, #token: 1348608, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.48, #queue-req: 0 | |
| [2026-06-26 20:10:08 DP2 TP2] Decode batch, #running-req: 32, #token: 1350336, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.90, #queue-req: 0 | |
| [2026-06-26 20:10:08 DP0 TP0] Decode batch, #running-req: 32, #token: 1351744, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.89, #queue-req: 0 | |
| [2026-06-26 20:10:08 DP1 TP1] Decode batch, #running-req: 32, #token: 1352512, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.90, #queue-req: 0 | |
| [2026-06-26 20:10:09 DP3 TP3] Decode batch, #running-req: 32, #token: 1350208, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.99, #queue-req: 0 | |
| [2026-06-26 20:10:10 DP2 TP2] Decode batch, #running-req: 32, #token: 1351872, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.02, #queue-req: 0 | |
| [2026-06-26 20:10:10 DP1 TP1] Decode batch, #running-req: 32, #token: 1353920, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.04, #queue-req: 0 | |
| [2026-06-26 20:10:10 DP0 TP0] Decode batch, #running-req: 32, #token: 1352832, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1111.04, #queue-req: 0 | |
| [2026-06-26 20:10:10 DP3 TP3] Decode batch, #running-req: 32, #token: 1351232, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.91, #queue-req: 0 | |
| [2026-06-26 20:10:11 DP0 TP0] Decode batch, #running-req: 32, #token: 1354368, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.74, #queue-req: 0 | |
| [2026-06-26 20:10:11 DP1 TP1] Decode batch, #running-req: 32, #token: 1355200, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.74, #queue-req: 0 | |
| [2026-06-26 20:10:11 DP2 TP2] Decode batch, #running-req: 32, #token: 1353088, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.74, #queue-req: 0 | |
| [2026-06-26 20:10:11 DP3 TP3] Decode batch, #running-req: 32, #token: 1352448, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.73, #queue-req: 0 | |
| [2026-06-26 20:10:12 DP0 TP0] Decode batch, #running-req: 32, #token: 1355456, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1108.46, #queue-req: 0 | |
| [2026-06-26 20:10:12 DP1 TP1] Decode batch, #running-req: 32, #token: 1356416, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1108.46, #queue-req: 0 | |
| [2026-06-26 20:10:12 DP2 TP2] Decode batch, #running-req: 32, #token: 1354304, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1108.46, #queue-req: 0 | |
| [2026-06-26 20:10:12 DP3 TP3] Decode batch, #running-req: 32, #token: 1353984, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1108.49, #queue-req: 0 | |
| [2026-06-26 20:10:13 DP1 TP1] Decode batch, #running-req: 32, #token: 1357888, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.39, #queue-req: 0 | |
| [2026-06-26 20:10:13 DP0 TP0] Decode batch, #running-req: 32, #token: 1356800, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.38, #queue-req: 0 | |
| [2026-06-26 20:10:13 DP2 TP2] Decode batch, #running-req: 32, #token: 1355776, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.38, #queue-req: 0 | |
| [2026-06-26 20:10:13 DP3 TP3] Decode batch, #running-req: 32, #token: 1354816, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.54, #queue-req: 0 | |
| [2026-06-26 20:10:14 DP0 TP0] Decode batch, #running-req: 32, #token: 1358208, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.82, #queue-req: 0 | |
| [2026-06-26 20:10:14 DP1 TP1] Decode batch, #running-req: 32, #token: 1358848, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.82, #queue-req: 0 | |
| [2026-06-26 20:10:14 DP2 TP2] Decode batch, #running-req: 32, #token: 1356928, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.84, #queue-req: 0 | |
| [2026-06-26 20:10:14 DP3 TP3] Decode batch, #running-req: 32, #token: 1356416, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1109.77, #queue-req: 0 | |
| [2026-06-26 20:10:15 DP1 TP1] Decode batch, #running-req: 32, #token: 1360320, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.28, #queue-req: 0 | |
| [2026-06-26 20:10:15 DP0 TP0] Decode batch, #running-req: 32, #token: 1359168, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.29, #queue-req: 0 | |
| [2026-06-26 20:10:15 DP2 TP2] Decode batch, #running-req: 32, #token: 1358272, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.26, #queue-req: 0 | |
| [2026-06-26 20:10:15 DP3 TP3] Decode batch, #running-req: 32, #token: 1357824, token usage: 0.76, cuda graph: True, gen throughput (token/s): 1110.19, #queue-req: 0 | |
| [2026-06-26 20:10:17 DP0 TP0] Decode batch, #running-req: 32, #token: 1360768, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:17 DP1 TP1] Decode batch, #running-req: 32, #token: 1361728, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.92, #queue-req: 0 | |
| [2026-06-26 20:10:17 DP2 TP2] Decode batch, #running-req: 32, #token: 1359680, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.92, #queue-req: 0 | |
| [2026-06-26 20:10:17 DP3 TP3] Decode batch, #running-req: 32, #token: 1358848, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.85, #queue-req: 0 | |
| [2026-06-26 20:10:18 DP0 TP0] Decode batch, #running-req: 32, #token: 1361984, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.66, #queue-req: 0 | |
| [2026-06-26 20:10:18 DP1 TP1] Decode batch, #running-req: 32, #token: 1362752, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.65, #queue-req: 0 | |
| [2026-06-26 20:10:18 DP2 TP2] Decode batch, #running-req: 32, #token: 1360576, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.67, #queue-req: 0 | |
| [2026-06-26 20:10:18 DP3 TP3] Decode batch, #running-req: 32, #token: 1360448, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.74, #queue-req: 0 | |
| [2026-06-26 20:10:19 DP0 TP0] Decode batch, #running-req: 32, #token: 1363072, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.28, #queue-req: 0 | |
| [2026-06-26 20:10:19 DP1 TP1] Decode batch, #running-req: 32, #token: 1364160, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.29, #queue-req: 0 | |
| [2026-06-26 20:10:19 DP2 TP2] Decode batch, #running-req: 32, #token: 1362112, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.27, #queue-req: 0 | |
| [2026-06-26 20:10:19 DP3 TP3] Decode batch, #running-req: 32, #token: 1361472, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.18, #queue-req: 0 | |
| [2026-06-26 20:10:20 DP1 TP1] Decode batch, #running-req: 32, #token: 1365440, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.70, #queue-req: 0 | |
| [2026-06-26 20:10:20 DP0 TP0] Decode batch, #running-req: 32, #token: 1364608, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.70, #queue-req: 0 | |
| [2026-06-26 20:10:20 DP2 TP2] Decode batch, #running-req: 32, #token: 1363328, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.71, #queue-req: 0 | |
| [2026-06-26 20:10:20 DP3 TP3] Decode batch, #running-req: 32, #token: 1362688, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.83, #queue-req: 0 | |
| [2026-06-26 20:10:21 DP0 TP0] Decode batch, #running-req: 32, #token: 1365696, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.83, #queue-req: 0 | |
| [2026-06-26 20:10:21 DP1 TP1] Decode batch, #running-req: 32, #token: 1366656, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.83, #queue-req: 0 | |
| [2026-06-26 20:10:21 DP2 TP2] Decode batch, #running-req: 32, #token: 1364544, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.82, #queue-req: 0 | |
| [2026-06-26 20:10:21 DP3 TP3] Decode batch, #running-req: 32, #token: 1364224, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.78, #queue-req: 0 | |
| [2026-06-26 20:10:22 DP1 TP1] Decode batch, #running-req: 32, #token: 1368128, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.19, #queue-req: 0 | |
| [2026-06-26 20:10:22 DP0 TP0] Decode batch, #running-req: 32, #token: 1367040, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.20, #queue-req: 0 | |
| [2026-06-26 20:10:22 DP2 TP2] Decode batch, #running-req: 32, #token: 1366016, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.20, #queue-req: 0 | |
| [2026-06-26 20:10:22 DP3 TP3] Decode batch, #running-req: 32, #token: 1365056, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.25, #queue-req: 0 | |
| [2026-06-26 20:10:23 DP1 TP1] Decode batch, #running-req: 32, #token: 1369088, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.73, #queue-req: 0 | |
| [2026-06-26 20:10:23 DP0 TP0] Decode batch, #running-req: 32, #token: 1368448, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.73, #queue-req: 0 | |
| [2026-06-26 20:10:23 DP2 TP2] Decode batch, #running-req: 32, #token: 1367168, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.72, #queue-req: 0 | |
| [2026-06-26 20:10:24 DP3 TP3] Decode batch, #running-req: 32, #token: 1366656, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1110.72, #queue-req: 0 | |
| [2026-06-26 20:10:25 DP1 TP1] Decode batch, #running-req: 32, #token: 1370560, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.20, #queue-req: 0 | |
| [2026-06-26 20:10:25 DP0 TP0] Decode batch, #running-req: 32, #token: 1369408, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.20, #queue-req: 0 | |
| [2026-06-26 20:10:25 DP2 TP2] Decode batch, #running-req: 32, #token: 1368512, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.20, #queue-req: 0 | |
| [2026-06-26 20:10:25 DP3 TP3] Decode batch, #running-req: 32, #token: 1368064, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.33, #queue-req: 0 | |
| [2026-06-26 20:10:26 DP1 TP1] Decode batch, #running-req: 32, #token: 1371968, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.74, #queue-req: 0 | |
| [2026-06-26 20:10:26 DP0 TP0] Decode batch, #running-req: 32, #token: 1371008, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.74, #queue-req: 0 | |
| [2026-06-26 20:10:26 DP2 TP2] Decode batch, #running-req: 32, #token: 1369920, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.75, #queue-req: 0 | |
| [2026-06-26 20:10:26 DP3 TP3] Decode batch, #running-req: 32, #token: 1369088, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.58, #queue-req: 0 | |
| [2026-06-26 20:10:27 DP2 TP2] Decode batch, #running-req: 32, #token: 1370816, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.77, #queue-req: 0 | |
| [2026-06-26 20:10:27 DP0 TP0] Decode batch, #running-req: 32, #token: 1372224, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.76, #queue-req: 0 | |
| [2026-06-26 20:10:27 DP1 TP1] Decode batch, #running-req: 32, #token: 1372992, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.76, #queue-req: 0 | |
| [2026-06-26 20:10:27 DP3 TP3] Decode batch, #running-req: 32, #token: 1370688, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1109.71, #queue-req: 0 | |
| [2026-06-26 20:10:28 DP2 TP2] Decode batch, #running-req: 32, #token: 1372352, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.54, #queue-req: 0 | |
| [2026-06-26 20:10:28 DP0 TP0] Decode batch, #running-req: 32, #token: 1373312, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.44, #queue-req: 0 | |
| [2026-06-26 20:10:28 DP1 TP1] Decode batch, #running-req: 32, #token: 1374400, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.40, #queue-req: 0 | |
| [2026-06-26 20:10:28 DP3 TP3] Decode batch, #running-req: 32, #token: 1371712, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.43, #queue-req: 0 | |
| [2026-06-26 20:10:29 DP0 TP0] Decode batch, #running-req: 32, #token: 1374848, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.90, #queue-req: 0 | |
| [2026-06-26 20:10:29 DP1 TP1] Decode batch, #running-req: 32, #token: 1375680, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:29 DP2 TP2] Decode batch, #running-req: 32, #token: 1373568, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1108.77, #queue-req: 0 | |
| [2026-06-26 20:10:29 DP3 TP3] Decode batch, #running-req: 32, #token: 1372928, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.93, #queue-req: 0 | |
| [2026-06-26 20:10:30 DP2 TP2] Decode batch, #running-req: 32, #token: 1374784, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.82, #queue-req: 0 | |
| [2026-06-26 20:10:30 DP1 TP1] Decode batch, #running-req: 32, #token: 1376896, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1107.82, #queue-req: 0 | |
| [2026-06-26 20:10:30 DP0 TP0] Decode batch, #running-req: 32, #token: 1375936, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.82, #queue-req: 0 | |
| [2026-06-26 20:10:30 DP3 TP3] Decode batch, #running-req: 32, #token: 1374464, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.75, #queue-req: 0 | |
| [2026-06-26 20:10:32 DP1 TP1] Decode batch, #running-req: 32, #token: 1378368, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1107.79, #queue-req: 0 | |
| [2026-06-26 20:10:32 DP0 TP0] Decode batch, #running-req: 32, #token: 1377280, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1107.79, #queue-req: 0 | |
| [2026-06-26 20:10:32 DP2 TP2] Decode batch, #running-req: 32, #token: 1376256, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1107.79, #queue-req: 0 | |
| [2026-06-26 20:10:32 DP3 TP3] Decode batch, #running-req: 32, #token: 1375296, token usage: 0.77, cuda graph: True, gen throughput (token/s): 1107.83, #queue-req: 0 | |
| [2026-06-26 20:10:33 DP1 TP1] Decode batch, #running-req: 32, #token: 1379328, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.66, #queue-req: 0 | |
| [2026-06-26 20:10:33 DP0 TP0] Decode batch, #running-req: 32, #token: 1378688, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.66, #queue-req: 0 | |
| [2026-06-26 20:10:33 DP2 TP2] Decode batch, #running-req: 32, #token: 1377408, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.64, #queue-req: 0 | |
| [2026-06-26 20:10:33 DP3 TP3] Decode batch, #running-req: 32, #token: 1376896, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.73, #queue-req: 0 | |
| [2026-06-26 20:10:34 DP1 TP1] Decode batch, #running-req: 32, #token: 1380800, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1111.45, #queue-req: 0 | |
| [2026-06-26 20:10:34 DP0 TP0] Decode batch, #running-req: 32, #token: 1379648, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1111.45, #queue-req: 0 | |
| [2026-06-26 20:10:34 DP2 TP2] Decode batch, #running-req: 32, #token: 1378752, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1111.49, #queue-req: 0 | |
| [2026-06-26 20:10:34 DP3 TP3] Decode batch, #running-req: 32, #token: 1378304, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1111.37, #queue-req: 0 | |
| [2026-06-26 20:10:35 DP1 TP1] Decode batch, #running-req: 32, #token: 1382208, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.47, #queue-req: 0 | |
| [2026-06-26 20:10:35 DP0 TP0] Decode batch, #running-req: 32, #token: 1381248, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.47, #queue-req: 0 | |
| [2026-06-26 20:10:35 DP2 TP2] Decode batch, #running-req: 32, #token: 1380160, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.45, #queue-req: 0 | |
| [2026-06-26 20:10:35 DP3 TP3] Decode batch, #running-req: 32, #token: 1379328, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1110.56, #queue-req: 0 | |
| [2026-06-26 20:10:36 DP0 TP0] Decode batch, #running-req: 32, #token: 1382464, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:36 DP1 TP1] Decode batch, #running-req: 32, #token: 1383232, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:36 DP2 TP2] Decode batch, #running-req: 32, #token: 1381056, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:36 DP3 TP3] Decode batch, #running-req: 32, #token: 1380928, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1108.93, #queue-req: 0 | |
| [2026-06-26 20:10:37 DP2 TP2] Decode batch, #running-req: 32, #token: 1382592, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1109.55, #queue-req: 0 | |
| [2026-06-26 20:10:37 DP1 TP1] Decode batch, #running-req: 32, #token: 1384640, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1109.53, #queue-req: 0 | |
| [2026-06-26 20:10:37 DP0 TP0] Decode batch, #running-req: 32, #token: 1383552, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1109.52, #queue-req: 0 | |
| [2026-06-26 20:10:37 DP3 TP3] Decode batch, #running-req: 32, #token: 1381952, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1109.43, #queue-req: 0 | |
| [2026-06-26 20:10:38 DP2 TP2] Decode batch, #running-req: 32, #token: 1383808, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1106.06, #queue-req: 0 | |
| [2026-06-26 20:10:38 DP0 TP0] Decode batch, #running-req: 32, #token: 1385088, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1106.09, #queue-req: 0 | |
| [2026-06-26 20:10:38 DP1 TP1] Decode batch, #running-req: 32, #token: 1385920, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1106.08, #queue-req: 0 | |
| [2026-06-26 20:10:39 DP3 TP3] Decode batch, #running-req: 32, #token: 1383168, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1106.02, #queue-req: 0 | |
| [2026-06-26 20:10:40 DP0 TP0] Decode batch, #running-req: 32, #token: 1386176, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1104.50, #queue-req: 0 | |
| [2026-06-26 20:10:40 DP1 TP1] Decode batch, #running-req: 32, #token: 1387136, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1104.49, #queue-req: 0 | |
| [2026-06-26 20:10:40 DP2 TP2] Decode batch, #running-req: 32, #token: 1385024, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1104.49, #queue-req: 0 | |
| [2026-06-26 20:10:40 DP3 TP3] Decode batch, #running-req: 32, #token: 1384704, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1104.41, #queue-req: 0 | |
| [2026-06-26 20:10:41 DP1 TP1] Decode batch, #running-req: 32, #token: 1388608, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1105.81, #queue-req: 0 | |
| [2026-06-26 20:10:41 DP0 TP0] Decode batch, #running-req: 32, #token: 1387520, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1105.79, #queue-req: 0 | |
| [2026-06-26 20:10:41 DP2 TP2] Decode batch, #running-req: 32, #token: 1386496, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1105.80, #queue-req: 0 | |
| [2026-06-26 20:10:41 DP3 TP3] Decode batch, #running-req: 32, #token: 1385536, token usage: 0.78, cuda graph: True, gen throughput (token/s): 1105.95, #queue-req: 0 | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47994 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47704 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47694 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47766 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47786 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47832 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47870 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47918 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47964 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48006 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48044 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48086 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48108 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48138 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48174 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48204 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48240 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48286 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48314 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48354 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48388 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48490 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48540 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48584 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48610 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48636 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48660 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48706 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48724 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48750 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48440 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47824 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47682 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47742 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47784 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47812 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47860 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47914 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47956 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47998 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48038 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48072 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48100 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48128 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48164 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48198 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48228 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48280 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48310 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48350 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48382 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48424 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48476 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48524 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48576 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48598 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48630 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48654 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48720 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48780 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48150 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47664 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47668 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47718 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47772 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47794 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47846 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47886 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47932 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47980 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48008 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48052 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48090 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48116 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48140 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48176 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48210 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48250 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48298 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48324 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48370 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48394 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48446 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48498 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48554 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48594 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48614 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48638 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48670 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48708 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48736 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48764 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48022 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47754 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47648 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47732 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47774 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47806 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47852 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47900 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47940 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:47992 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48020 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48058 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48094 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48118 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48160 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48186 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48214 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48266 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48300 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48340 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48376 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48410 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48462 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48514 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48570 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48596 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48626 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48652 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48686 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48714 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48738 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:42] INFO: 127.0.0.1:48776 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:10:44 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41088, token usage: 0.03, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 146.76 | |
| [2026-06-26 20:10:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.06, #running-req: 0, #queue-req: 5, #pending-token: 0, cuda graph: False, input throughput (token/s): 781.53 | |
| [2026-06-26 20:10:52 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.06, #running-req: 0, #queue-req: 14, #pending-token: 151286, cuda graph: False, input throughput (token/s): 735.53 | |
| [2026-06-26 20:10:52 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41344, token usage: 0.09, #running-req: 1, #queue-req: 14, #pending-token: 304220, cuda graph: False, input throughput (token/s): 1257.05 | |
| [2026-06-26 20:10:52 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.09, #running-req: 1, #queue-req: 14, #pending-token: 252035, cuda graph: False, input throughput (token/s): 12214.20 | |
| [2026-06-26 20:10:52 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.06, #running-req: 0, #queue-req: 15, #pending-token: 151100, cuda graph: False, input throughput (token/s): 733.65 | |
| [2026-06-26 20:10:55 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.09, #running-req: 1, #queue-req: 27, #pending-token: 707117, cuda graph: False, input throughput (token/s): 13091.01 | |
| [2026-06-26 20:10:55 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.09, #running-req: 1, #queue-req: 27, #pending-token: 755055, cuda graph: False, input throughput (token/s): 13075.65 | |
| [2026-06-26 20:10:55 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.11, #running-req: 2, #queue-req: 27, #pending-token: 705913, cuda graph: False, input throughput (token/s): 13107.83 | |
| [2026-06-26 20:10:57 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.11, #running-req: 2, #queue-req: 27, #pending-token: 706994, cuda graph: False, input throughput (token/s): 10083.94 | |
| [2026-06-26 20:10:59 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.11, #running-req: 2, #queue-req: 28, #pending-token: 1363265, cuda graph: False, input throughput (token/s): 14852.38 | |
| [2026-06-26 20:10:59 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.11, #running-req: 2, #queue-req: 28, #pending-token: 1360637, cuda graph: False, input throughput (token/s): 14777.14 | |
| [2026-06-26 20:10:59 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 51520, #cached-token: 0, token usage: 0.14, #running-req: 3, #queue-req: 27, #pending-token: 1362339, cuda graph: False, input throughput (token/s): 22893.97 | |
| [2026-06-26 20:10:59 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41280, token usage: 0.14, #running-req: 3, #queue-req: 27, #pending-token: 1361624, cuda graph: False, input throughput (token/s): 2688.40 | |
| [2026-06-26 20:11:03 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.14, #running-req: 3, #queue-req: 27, #pending-token: 1413486, cuda graph: False, input throughput (token/s): 12221.55 | |
| [2026-06-26 20:11:03 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41280, token usage: 0.17, #running-req: 4, #queue-req: 26, #pending-token: 1361739, cuda graph: False, input throughput (token/s): 2230.61 | |
| [2026-06-26 20:11:03 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41408, token usage: 0.14, #running-req: 3, #queue-req: 27, #pending-token: 1410616, cuda graph: False, input throughput (token/s): 2230.43 | |
| [2026-06-26 20:11:04 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.17, #running-req: 4, #queue-req: 26, #pending-token: 1362050, cuda graph: False, input throughput (token/s): 9420.69 | |
| [2026-06-26 20:11:07 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 51200, #cached-token: 0, token usage: 0.20, #running-req: 5, #queue-req: 25, #pending-token: 1310865, cuda graph: False, input throughput (token/s): 19332.11 | |
| [2026-06-26 20:11:07 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.20, #running-req: 5, #queue-req: 25, #pending-token: 1311385, cuda graph: False, input throughput (token/s): 12969.56 | |
| [2026-06-26 20:11:07 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.17, #running-req: 4, #queue-req: 26, #pending-token: 1360342, cuda graph: False, input throughput (token/s): 12953.14 | |
| [2026-06-26 20:11:07 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50752, #cached-token: 0, token usage: 0.17, #running-req: 4, #queue-req: 26, #pending-token: 1362749, cuda graph: False, input throughput (token/s): 13067.45 | |
| [2026-06-26 20:11:10 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.20, #running-req: 5, #queue-req: 25, #pending-token: 1312471, cuda graph: False, input throughput (token/s): 16186.97 | |
| [2026-06-26 20:11:10 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.20, #running-req: 5, #queue-req: 25, #pending-token: 1310132, cuda graph: False, input throughput (token/s): 16165.92 | |
| [2026-06-26 20:11:10 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.23, #running-req: 6, #queue-req: 24, #pending-token: 1260983, cuda graph: False, input throughput (token/s): 16226.88 | |
| [2026-06-26 20:11:10 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 18304, #cached-token: 32000, token usage: 0.23, #running-req: 6, #queue-req: 24, #pending-token: 1260594, cuda graph: False, input throughput (token/s): 5889.29 | |
| [2026-06-26 20:11:14 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41216, token usage: 0.23, #running-req: 6, #queue-req: 24, #pending-token: 1262064, cuda graph: False, input throughput (token/s): 2457.68 | |
| [2026-06-26 20:11:14 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41152, token usage: 0.26, #running-req: 7, #queue-req: 23, #pending-token: 1210290, cuda graph: False, input throughput (token/s): 2440.64 | |
| [2026-06-26 20:11:14 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.26, #running-req: 7, #queue-req: 23, #pending-token: 1210726, cuda graph: False, input throughput (token/s): 13414.59 | |
| [2026-06-26 20:11:14 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41152, token usage: 0.23, #running-req: 6, #queue-req: 24, #pending-token: 1259855, cuda graph: False, input throughput (token/s): 2440.31 | |
| [2026-06-26 20:11:18 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.26, #running-req: 7, #queue-req: 23, #pending-token: 1211809, cuda graph: False, input throughput (token/s): 11974.58 | |
| [2026-06-26 20:11:18 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.26, #running-req: 7, #queue-req: 23, #pending-token: 1209325, cuda graph: False, input throughput (token/s): 12036.78 | |
| [2026-06-26 20:11:18 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.28, #running-req: 8, #queue-req: 22, #pending-token: 1159960, cuda graph: False, input throughput (token/s): 11989.29 | |
| [2026-06-26 20:11:18 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.28, #running-req: 8, #queue-req: 22, #pending-token: 1160271, cuda graph: False, input throughput (token/s): 12019.95 | |
| [2026-06-26 20:11:22 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.31, #running-req: 9, #queue-req: 21, #pending-token: 1109728, cuda graph: False, input throughput (token/s): 13400.55 | |
| [2026-06-26 20:11:22 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.31, #running-req: 9, #queue-req: 21, #pending-token: 1109805, cuda graph: False, input throughput (token/s): 13468.67 | |
| [2026-06-26 20:11:22 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.28, #running-req: 8, #queue-req: 22, #pending-token: 1158940, cuda graph: False, input throughput (token/s): 13450.76 | |
| [2026-06-26 20:11:22 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 51072, #cached-token: 0, token usage: 0.28, #running-req: 8, #queue-req: 22, #pending-token: 1160745, cuda graph: False, input throughput (token/s): 13621.34 | |
| [2026-06-26 20:11:25 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.31, #running-req: 9, #queue-req: 21, #pending-token: 1108612, cuda graph: False, input throughput (token/s): 13695.82 | |
| [2026-06-26 20:11:25 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.34, #running-req: 10, #queue-req: 20, #pending-token: 1059428, cuda graph: False, input throughput (token/s): 13713.11 | |
| [2026-06-26 20:11:25 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 41408, token usage: 0.31, #running-req: 9, #queue-req: 21, #pending-token: 1110025, cuda graph: False, input throughput (token/s): 2540.74 | |
| [2026-06-26 20:11:25 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.34, #running-req: 10, #queue-req: 20, #pending-token: 1059330, cuda graph: False, input throughput (token/s): 13712.87 | |
| [2026-06-26 20:11:29 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.34, #running-req: 10, #queue-req: 20, #pending-token: 1059750, cuda graph: False, input throughput (token/s): 14876.31 | |
| [2026-06-26 20:11:29 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.34, #running-req: 10, #queue-req: 20, #pending-token: 1058388, cuda graph: False, input throughput (token/s): 14857.19 | |
| [2026-06-26 20:11:29 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.37, #running-req: 11, #queue-req: 19, #pending-token: 1008988, cuda graph: False, input throughput (token/s): 14894.69 | |
| [2026-06-26 20:11:29 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41344, token usage: 0.37, #running-req: 11, #queue-req: 19, #pending-token: 1008910, cuda graph: False, input throughput (token/s): 2725.26 | |
| [2026-06-26 20:11:32 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.37, #running-req: 11, #queue-req: 19, #pending-token: 1009481, cuda graph: False, input throughput (token/s): 14311.78 | |
| [2026-06-26 20:11:32 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.37, #running-req: 11, #queue-req: 19, #pending-token: 1007899, cuda graph: False, input throughput (token/s): 14366.32 | |
| [2026-06-26 20:11:32 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41408, token usage: 0.40, #running-req: 12, #queue-req: 18, #pending-token: 958399, cuda graph: False, input throughput (token/s): 2622.06 | |
| [2026-06-26 20:11:32 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41408, token usage: 0.40, #running-req: 12, #queue-req: 18, #pending-token: 958314, cuda graph: False, input throughput (token/s): 2622.10 | |
| [2026-06-26 20:11:36 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.43, #running-req: 13, #queue-req: 17, #pending-token: 908031, cuda graph: False, input throughput (token/s): 12270.69 | |
| [2026-06-26 20:11:36 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.40, #running-req: 12, #queue-req: 18, #pending-token: 957605, cuda graph: False, input throughput (token/s): 12254.86 | |
| [2026-06-26 20:11:36 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 41408, token usage: 0.43, #running-req: 13, #queue-req: 17, #pending-token: 907669, cuda graph: False, input throughput (token/s): 2260.76 | |
| [2026-06-26 20:11:36 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.40, #running-req: 12, #queue-req: 18, #pending-token: 959175, cuda graph: False, input throughput (token/s): 12270.26 | |
| [2026-06-26 20:11:40 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50176, #cached-token: 0, token usage: 0.43, #running-req: 13, #queue-req: 17, #pending-token: 907468, cuda graph: False, input throughput (token/s): 13598.61 | |
| [2026-06-26 20:11:40 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.46, #running-req: 14, #queue-req: 16, #pending-token: 857779, cuda graph: False, input throughput (token/s): 13619.25 | |
| [2026-06-26 20:11:40 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.45, #running-req: 14, #queue-req: 16, #pending-token: 857361, cuda graph: False, input throughput (token/s): 13636.76 | |
| [2026-06-26 20:11:40 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.43, #running-req: 13, #queue-req: 17, #pending-token: 908696, cuda graph: False, input throughput (token/s): 13670.98 | |
| [2026-06-26 20:11:43 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.48, #running-req: 15, #queue-req: 15, #pending-token: 807489, cuda graph: False, input throughput (token/s): 14889.28 | |
| [2026-06-26 20:11:43 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50944, #cached-token: 0, token usage: 0.46, #running-req: 14, #queue-req: 16, #pending-token: 857814, cuda graph: False, input throughput (token/s): 15079.42 | |
| [2026-06-26 20:11:43 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.48, #running-req: 15, #queue-req: 15, #pending-token: 806913, cuda graph: False, input throughput (token/s): 14928.90 | |
| [2026-06-26 20:11:43 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41152, token usage: 0.45, #running-req: 14, #queue-req: 16, #pending-token: 857160, cuda graph: False, input throughput (token/s): 2721.37 | |
| [2026-06-26 20:11:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.51, #running-req: 16, #queue-req: 14, #pending-token: 756677, cuda graph: False, input throughput (token/s): 14872.30 | |
| [2026-06-26 20:11:47 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 41152, token usage: 0.51, #running-req: 16, #queue-req: 14, #pending-token: 757116, cuda graph: False, input throughput (token/s): 2730.68 | |
| [2026-06-26 20:11:47 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41280, token usage: 0.48, #running-req: 15, #queue-req: 15, #pending-token: 807328, cuda graph: False, input throughput (token/s): 2711.81 | |
| [2026-06-26 20:11:47 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.48, #running-req: 15, #queue-req: 15, #pending-token: 806859, cuda graph: False, input throughput (token/s): 14818.36 | |
| [2026-06-26 20:11:51 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.51, #running-req: 16, #queue-req: 14, #pending-token: 756933, cuda graph: False, input throughput (token/s): 12008.53 | |
| [2026-06-26 20:11:51 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.51, #running-req: 16, #queue-req: 14, #pending-token: 756406, cuda graph: False, input throughput (token/s): 12025.14 | |
| [2026-06-26 20:11:51 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.54, #running-req: 17, #queue-req: 13, #pending-token: 706575, cuda graph: False, input throughput (token/s): 12038.62 | |
| [2026-06-26 20:11:51 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41152, token usage: 0.54, #running-req: 17, #queue-req: 13, #pending-token: 706402, cuda graph: False, input throughput (token/s): 2170.62 | |
| [2026-06-26 20:11:55 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.54, #running-req: 17, #queue-req: 13, #pending-token: 706419, cuda graph: False, input throughput (token/s): 13520.83 | |
| [2026-06-26 20:11:55 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50752, #cached-token: 0, token usage: 0.54, #running-req: 17, #queue-req: 13, #pending-token: 705711, cuda graph: False, input throughput (token/s): 13571.98 | |
| [2026-06-26 20:11:55 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.57, #running-req: 18, #queue-req: 12, #pending-token: 656194, cuda graph: False, input throughput (token/s): 13486.66 | |
| [2026-06-26 20:11:55 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.57, #running-req: 18, #queue-req: 12, #pending-token: 655934, cuda graph: False, input throughput (token/s): 13503.97 | |
| [2026-06-26 20:11:58 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.60, #running-req: 19, #queue-req: 11, #pending-token: 605580, cuda graph: False, input throughput (token/s): 14923.87 | |
| [2026-06-26 20:11:58 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.60, #running-req: 19, #queue-req: 11, #pending-token: 605911, cuda graph: False, input throughput (token/s): 14904.67 | |
| [2026-06-26 20:11:58 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.57, #running-req: 18, #queue-req: 12, #pending-token: 655474, cuda graph: False, input throughput (token/s): 14885.66 | |
| [2026-06-26 20:11:58 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 42112, token usage: 0.57, #running-req: 18, #queue-req: 12, #pending-token: 655171, cuda graph: False, input throughput (token/s): 2711.61 | |
| [2026-06-26 20:12:02 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.62, #running-req: 20, #queue-req: 10, #pending-token: 555402, cuda graph: False, input throughput (token/s): 12734.57 | |
| [2026-06-26 20:12:02 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 41344, token usage: 0.63, #running-req: 20, #queue-req: 10, #pending-token: 555338, cuda graph: False, input throughput (token/s): 2352.20 | |
| [2026-06-26 20:12:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.60, #running-req: 19, #queue-req: 11, #pending-token: 604903, cuda graph: False, input throughput (token/s): 12750.62 | |
| [2026-06-26 20:12:02 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9088, #cached-token: 41280, token usage: 0.60, #running-req: 19, #queue-req: 11, #pending-token: 605110, cuda graph: False, input throughput (token/s): 2303.54 | |
| [2026-06-26 20:12:06 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.65, #running-req: 21, #queue-req: 9, #pending-token: 504969, cuda graph: False, input throughput (token/s): 14785.38 | |
| [2026-06-26 20:12:06 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.63, #running-req: 20, #queue-req: 10, #pending-token: 554475, cuda graph: False, input throughput (token/s): 14784.43 | |
| [2026-06-26 20:12:06 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.62, #running-req: 20, #queue-req: 10, #pending-token: 554589, cuda graph: False, input throughput (token/s): 14821.78 | |
| [2026-06-26 20:12:06 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.65, #running-req: 21, #queue-req: 9, #pending-token: 504895, cuda graph: False, input throughput (token/s): 14819.74 | |
| [2026-06-26 20:12:09 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.65, #running-req: 21, #queue-req: 9, #pending-token: 504080, cuda graph: False, input throughput (token/s): 14375.97 | |
| [2026-06-26 20:12:09 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.68, #running-req: 22, #queue-req: 8, #pending-token: 454747, cuda graph: False, input throughput (token/s): 14320.38 | |
| [2026-06-26 20:12:09 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41152, token usage: 0.68, #running-req: 22, #queue-req: 8, #pending-token: 454609, cuda graph: False, input throughput (token/s): 2609.10 | |
| [2026-06-26 20:12:09 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 41408, token usage: 0.65, #running-req: 21, #queue-req: 9, #pending-token: 504017, cuda graph: False, input throughput (token/s): 2627.01 | |
| [2026-06-26 20:12:13 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.68, #running-req: 22, #queue-req: 8, #pending-token: 453807, cuda graph: False, input throughput (token/s): 12207.43 | |
| [2026-06-26 20:12:13 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.68, #running-req: 22, #queue-req: 8, #pending-token: 453476, cuda graph: False, input throughput (token/s): 12269.80 | |
| [2026-06-26 20:12:13 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.71, #running-req: 23, #queue-req: 7, #pending-token: 404288, cuda graph: False, input throughput (token/s): 12222.97 | |
| [2026-06-26 20:12:13 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41216, token usage: 0.71, #running-req: 23, #queue-req: 7, #pending-token: 404419, cuda graph: False, input throughput (token/s): 2220.87 | |
| [2026-06-26 20:12:17 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50624, #cached-token: 0, token usage: 0.71, #running-req: 23, #queue-req: 7, #pending-token: 403221, cuda graph: False, input throughput (token/s): 12792.45 | |
| [2026-06-26 20:12:17 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.71, #running-req: 23, #queue-req: 7, #pending-token: 403073, cuda graph: False, input throughput (token/s): 12744.05 | |
| [2026-06-26 20:12:17 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50496, #cached-token: 0, token usage: 0.74, #running-req: 24, #queue-req: 6, #pending-token: 353833, cuda graph: False, input throughput (token/s): 12759.85 | |
| [2026-06-26 20:12:17 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50880, #cached-token: 0, token usage: 0.74, #running-req: 24, #queue-req: 6, #pending-token: 353588, cuda graph: False, input throughput (token/s): 12855.82 | |
| [2026-06-26 20:12:21 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50624, #cached-token: 0, token usage: 0.77, #running-req: 25, #queue-req: 5, #pending-token: 302990, cuda graph: False, input throughput (token/s): 11796.90 | |
| [2026-06-26 20:12:21 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.74, #running-req: 24, #queue-req: 6, #pending-token: 352686, cuda graph: False, input throughput (token/s): 11780.43 | |
| [2026-06-26 20:12:21 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.74, #running-req: 24, #queue-req: 6, #pending-token: 352771, cuda graph: False, input throughput (token/s): 11720.81 | |
| [2026-06-26 20:12:21 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50816, #cached-token: 0, token usage: 0.77, #running-req: 25, #queue-req: 5, #pending-token: 303076, cuda graph: False, input throughput (token/s): 11840.24 | |
| [2026-06-26 20:12:26 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.77, #running-req: 25, #queue-req: 5, #pending-token: 302290, cuda graph: False, input throughput (token/s): 11678.15 | |
| [2026-06-26 20:12:26 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50240, #cached-token: 0, token usage: 0.77, #running-req: 25, #queue-req: 5, #pending-token: 302534, cuda graph: False, input throughput (token/s): 11633.51 | |
| [2026-06-26 20:12:26 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.80, #running-req: 26, #queue-req: 4, #pending-token: 252572, cuda graph: False, input throughput (token/s): 11707.78 | |
| [2026-06-26 20:12:27 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.80, #running-req: 26, #queue-req: 4, #pending-token: 252573, cuda graph: False, input throughput (token/s): 9133.93 | |
| [2026-06-26 20:12:29 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50624, #cached-token: 0, token usage: 0.80, #running-req: 26, #queue-req: 4, #pending-token: 251729, cuda graph: False, input throughput (token/s): 13319.95 | |
| [2026-06-26 20:12:29 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.82, #running-req: 27, #queue-req: 3, #pending-token: 202266, cuda graph: False, input throughput (token/s): 13252.96 | |
| [2026-06-26 20:12:30 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 51200, #cached-token: 0, token usage: 0.82, #running-req: 27, #queue-req: 3, #pending-token: 201427, cuda graph: False, input throughput (token/s): 19704.56 | |
| [2026-06-26 20:12:30 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.79, #running-req: 26, #queue-req: 4, #pending-token: 252175, cuda graph: False, input throughput (token/s): 13250.32 | |
| [2026-06-26 20:12:34 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.83, #running-req: 27, #queue-req: 3, #pending-token: 201349, cuda graph: False, input throughput (token/s): 12242.50 | |
| [2026-06-26 20:12:34 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 41152, token usage: 0.82, #running-req: 27, #queue-req: 3, #pending-token: 201884, cuda graph: False, input throughput (token/s): 2221.99 | |
| [2026-06-26 20:12:34 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.85, #running-req: 28, #queue-req: 2, #pending-token: 151117, cuda graph: False, input throughput (token/s): 12228.53 | |
| [2026-06-26 20:12:34 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50880, #cached-token: 0, token usage: 0.85, #running-req: 28, #queue-req: 2, #pending-token: 151413, cuda graph: False, input throughput (token/s): 12349.26 | |
| [2026-06-26 20:12:38 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.85, #running-req: 28, #queue-req: 2, #pending-token: 150950, cuda graph: False, input throughput (token/s): 12064.95 | |
| [2026-06-26 20:12:38 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.88, #running-req: 29, #queue-req: 1, #pending-token: 100845, cuda graph: False, input throughput (token/s): 12034.50 | |
| [2026-06-26 20:12:38 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50560, #cached-token: 0, token usage: 0.88, #running-req: 29, #queue-req: 1, #pending-token: 100889, cuda graph: False, input throughput (token/s): 12097.12 | |
| [2026-06-26 20:12:38 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50432, #cached-token: 0, token usage: 0.85, #running-req: 28, #queue-req: 2, #pending-token: 151491, cuda graph: False, input throughput (token/s): 12063.96 | |
| [2026-06-26 20:12:42 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50944, #cached-token: 0, token usage: 0.88, #running-req: 29, #queue-req: 1, #pending-token: 100560, cuda graph: False, input throughput (token/s): 12225.80 | |
| [2026-06-26 20:12:42 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50624, #cached-token: 0, token usage: 0.91, #running-req: 30, #queue-req: 0, #pending-token: 50251, cuda graph: False, input throughput (token/s): 12147.69 | |
| [2026-06-26 20:12:42 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.91, #running-req: 30, #queue-req: 0, #pending-token: 50578, cuda graph: False, input throughput (token/s): 12086.26 | |
| [2026-06-26 20:12:42 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.88, #running-req: 29, #queue-req: 1, #pending-token: 100670, cuda graph: False, input throughput (token/s): 12069.25 | |
| [2026-06-26 20:12:45 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.91, #running-req: 30, #queue-req: 0, #pending-token: 50362, cuda graph: False, input throughput (token/s): 15316.14 | |
| [2026-06-26 20:12:45 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.91, #running-req: 30, #queue-req: 0, #pending-token: 50283, cuda graph: False, input throughput (token/s): 15294.13 | |
| [2026-06-26 20:12:45 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.91, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 14966.90 | |
| [2026-06-26 20:12:45 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 50624, #cached-token: 0, token usage: 0.91, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15061.93 | |
| [2026-06-26 20:12:46 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 50304, #cached-token: 0, token usage: 0.91, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 48272.07 | |
| [2026-06-26 20:12:46 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 50368, #cached-token: 0, token usage: 0.91, #running-req: 31, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 48333.01 | |
| [2026-06-26 20:12:46 DP2 TP2] Decode batch, #running-req: 32, #token: 1616448, token usage: 0.91, cuda graph: True, gen throughput (token/s): 9.94, #queue-req: 0 | |
| [2026-06-26 20:12:46 DP0 TP0] Decode batch, #running-req: 32, #token: 1613376, token usage: 0.91, cuda graph: True, gen throughput (token/s): 10.18, #queue-req: 0 | |
| [2026-06-26 20:12:46 DP1 TP1] Decode batch, #running-req: 32, #token: 1616512, token usage: 0.91, cuda graph: True, gen throughput (token/s): 10.18, #queue-req: 0 | |
| [2026-06-26 20:12:46 DP3 TP3] Decode batch, #running-req: 32, #token: 1615232, token usage: 0.91, cuda graph: True, gen throughput (token/s): 10.19, #queue-req: 0 | |
| [2026-06-26 20:12:48 DP2 TP2] Decode batch, #running-req: 32, #token: 1617728, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1128.03, #queue-req: 0 | |
| [2026-06-26 20:12:48 DP3 TP3] Decode batch, #running-req: 32, #token: 1616640, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1127.16, #queue-req: 0 | |
| [2026-06-26 20:12:48 DP0 TP0] Decode batch, #running-req: 32, #token: 1614784, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1127.15, #queue-req: 0 | |
| [2026-06-26 20:12:48 DP1 TP1] Decode batch, #running-req: 32, #token: 1617984, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1127.15, #queue-req: 0 | |
| [2026-06-26 20:12:49 DP2 TP2] Decode batch, #running-req: 32, #token: 1619008, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1112.94, #queue-req: 0 | |
| [2026-06-26 20:12:49 DP3 TP3] Decode batch, #running-req: 32, #token: 1617536, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1112.63, #queue-req: 0 | |
| [2026-06-26 20:12:49 DP1 TP1] Decode batch, #running-req: 32, #token: 1618880, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1112.61, #queue-req: 0 | |
| [2026-06-26 20:12:49 DP0 TP0] Decode batch, #running-req: 32, #token: 1615936, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1112.61, #queue-req: 0 | |
| [2026-06-26 20:12:50 DP2 TP2] Decode batch, #running-req: 32, #token: 1620352, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.70, #queue-req: 0 | |
| [2026-06-26 20:12:50 DP3 TP3] Decode batch, #running-req: 32, #token: 1619200, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.65, #queue-req: 0 | |
| [2026-06-26 20:12:50 DP0 TP0] Decode batch, #running-req: 32, #token: 1617088, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.67, #queue-req: 0 | |
| [2026-06-26 20:12:50 DP1 TP1] Decode batch, #running-req: 32, #token: 1620480, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.66, #queue-req: 0 | |
| [2026-06-26 20:12:51 DP2 TP2] Decode batch, #running-req: 32, #token: 1621760, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.46, #queue-req: 0 | |
| [2026-06-26 20:12:51 DP3 TP3] Decode batch, #running-req: 32, #token: 1620096, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.52, #queue-req: 0 | |
| [2026-06-26 20:12:51 DP1 TP1] Decode batch, #running-req: 32, #token: 1621888, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.53, #queue-req: 0 | |
| [2026-06-26 20:12:51 DP0 TP0] Decode batch, #running-req: 32, #token: 1618560, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1109.51, #queue-req: 0 | |
| [2026-06-26 20:12:52 DP2 TP2] Decode batch, #running-req: 32, #token: 1622848, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.31, #queue-req: 0 | |
| [2026-06-26 20:12:52 DP3 TP3] Decode batch, #running-req: 32, #token: 1621568, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.23, #queue-req: 0 | |
| [2026-06-26 20:12:52 DP1 TP1] Decode batch, #running-req: 32, #token: 1622848, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.23, #queue-req: 0 | |
| [2026-06-26 20:12:52 DP0 TP0] Decode batch, #running-req: 32, #token: 1619840, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.24, #queue-req: 0 | |
| [2026-06-26 20:12:53 DP2 TP2] Decode batch, #running-req: 32, #token: 1624256, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1108.04, #queue-req: 0 | |
| [2026-06-26 20:12:53 DP3 TP3] Decode batch, #running-req: 32, #token: 1622976, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1108.05, #queue-req: 0 | |
| [2026-06-26 20:12:53 DP1 TP1] Decode batch, #running-req: 32, #token: 1624320, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1108.05, #queue-req: 0 | |
| [2026-06-26 20:12:53 DP0 TP0] Decode batch, #running-req: 32, #token: 1621056, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1108.05, #queue-req: 0 | |
| [2026-06-26 20:12:55 DP2 TP2] Decode batch, #running-req: 32, #token: 1625344, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1107.82, #queue-req: 0 | |
| [2026-06-26 20:12:55 DP3 TP3] Decode batch, #running-req: 32, #token: 1624064, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.77, #queue-req: 0 | |
| [2026-06-26 20:12:55 DP0 TP0] Decode batch, #running-req: 32, #token: 1622464, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1107.76, #queue-req: 0 | |
| [2026-06-26 20:12:55 DP1 TP1] Decode batch, #running-req: 32, #token: 1625472, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1107.77, #queue-req: 0 | |
| [2026-06-26 20:12:56 DP2 TP2] Decode batch, #running-req: 32, #token: 1626688, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.29, #queue-req: 0 | |
| [2026-06-26 20:12:56 DP3 TP3] Decode batch, #running-req: 32, #token: 1625472, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.55, #queue-req: 0 | |
| [2026-06-26 20:12:56 DP0 TP0] Decode batch, #running-req: 32, #token: 1623616, token usage: 0.91, cuda graph: True, gen throughput (token/s): 1110.57, #queue-req: 0 | |
| [2026-06-26 20:12:56 DP1 TP1] Decode batch, #running-req: 32, #token: 1626752, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.56, #queue-req: 0 | |
| [2026-06-26 20:12:57 DP2 TP2] Decode batch, #running-req: 32, #token: 1627968, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1113.94, #queue-req: 0 | |
| [2026-06-26 20:12:57 DP3 TP3] Decode batch, #running-req: 32, #token: 1626880, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1113.91, #queue-req: 0 | |
| [2026-06-26 20:12:57 DP1 TP1] Decode batch, #running-req: 32, #token: 1628224, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1113.89, #queue-req: 0 | |
| [2026-06-26 20:12:57 DP0 TP0] Decode batch, #running-req: 32, #token: 1625024, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1113.90, #queue-req: 0 | |
| [2026-06-26 20:12:58 DP2 TP2] Decode batch, #running-req: 32, #token: 1629248, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1114.65, #queue-req: 0 | |
| [2026-06-26 20:12:58 DP3 TP3] Decode batch, #running-req: 32, #token: 1627776, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1114.75, #queue-req: 0 | |
| [2026-06-26 20:12:58 DP1 TP1] Decode batch, #running-req: 32, #token: 1629120, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1114.75, #queue-req: 0 | |
| [2026-06-26 20:12:58 DP0 TP0] Decode batch, #running-req: 32, #token: 1626176, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1114.74, #queue-req: 0 | |
| [2026-06-26 20:12:59 DP2 TP2] Decode batch, #running-req: 32, #token: 1630592, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.45, #queue-req: 0 | |
| [2026-06-26 20:12:59 DP3 TP3] Decode batch, #running-req: 32, #token: 1629440, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.40, #queue-req: 0 | |
| [2026-06-26 20:12:59 DP0 TP0] Decode batch, #running-req: 32, #token: 1627328, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.38, #queue-req: 0 | |
| [2026-06-26 20:12:59 DP1 TP1] Decode batch, #running-req: 32, #token: 1630720, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.39, #queue-req: 0 | |
| [2026-06-26 20:13:00 DP2 TP2] Decode batch, #running-req: 32, #token: 1632000, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.07, #queue-req: 0 | |
| [2026-06-26 20:13:00 DP3 TP3] Decode batch, #running-req: 32, #token: 1630336, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.08, #queue-req: 0 | |
| [2026-06-26 20:13:00 DP1 TP1] Decode batch, #running-req: 32, #token: 1632128, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.10, #queue-req: 0 | |
| [2026-06-26 20:13:00 DP0 TP0] Decode batch, #running-req: 32, #token: 1628800, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.11, #queue-req: 0 | |
| [2026-06-26 20:13:01 DP2 TP2] Decode batch, #running-req: 32, #token: 1633088, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.09, #queue-req: 0 | |
| [2026-06-26 20:13:01 DP1 TP1] Decode batch, #running-req: 32, #token: 1633088, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.18, #queue-req: 0 | |
| [2026-06-26 20:13:01 DP0 TP0] Decode batch, #running-req: 32, #token: 1630080, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.18, #queue-req: 0 | |
| [2026-06-26 20:13:01 DP3 TP3] Decode batch, #running-req: 32, #token: 1631808, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.17, #queue-req: 0 | |
| [2026-06-26 20:13:03 DP2 TP2] Decode batch, #running-req: 32, #token: 1634496, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.08, #queue-req: 0 | |
| [2026-06-26 20:13:03 DP0 TP0] Decode batch, #running-req: 32, #token: 1631296, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.03, #queue-req: 0 | |
| [2026-06-26 20:13:03 DP1 TP1] Decode batch, #running-req: 32, #token: 1634560, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.04, #queue-req: 0 | |
| [2026-06-26 20:13:03 DP3 TP3] Decode batch, #running-req: 32, #token: 1633216, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.01, #queue-req: 0 | |
| [2026-06-26 20:13:04 DP2 TP2] Decode batch, #running-req: 32, #token: 1635584, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.80, #queue-req: 0 | |
| [2026-06-26 20:13:04 DP1 TP1] Decode batch, #running-req: 32, #token: 1635712, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.87, #queue-req: 0 | |
| [2026-06-26 20:13:04 DP0 TP0] Decode batch, #running-req: 32, #token: 1632704, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.87, #queue-req: 0 | |
| [2026-06-26 20:13:04 DP3 TP3] Decode batch, #running-req: 32, #token: 1634304, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.87, #queue-req: 0 | |
| [2026-06-26 20:13:05 DP2 TP2] Decode batch, #running-req: 32, #token: 1636928, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.30, #queue-req: 0 | |
| [2026-06-26 20:13:05 DP0 TP0] Decode batch, #running-req: 32, #token: 1633856, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.09, #queue-req: 0 | |
| [2026-06-26 20:13:05 DP1 TP1] Decode batch, #running-req: 32, #token: 1636992, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.10, #queue-req: 0 | |
| [2026-06-26 20:13:05 DP3 TP3] Decode batch, #running-req: 32, #token: 1635712, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.13, #queue-req: 0 | |
| [2026-06-26 20:13:06 DP2 TP2] Decode batch, #running-req: 32, #token: 1638208, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.11, #queue-req: 0 | |
| [2026-06-26 20:13:06 DP3 TP3] Decode batch, #running-req: 32, #token: 1637120, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.32, #queue-req: 0 | |
| [2026-06-26 20:13:06 DP1 TP1] Decode batch, #running-req: 32, #token: 1638464, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.30, #queue-req: 0 | |
| [2026-06-26 20:13:06 DP0 TP0] Decode batch, #running-req: 32, #token: 1635264, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1118.32, #queue-req: 0 | |
| [2026-06-26 20:13:07 DP2 TP2] Decode batch, #running-req: 32, #token: 1639488, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.23, #queue-req: 0 | |
| [2026-06-26 20:13:07 DP3 TP3] Decode batch, #running-req: 32, #token: 1638016, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.00, #queue-req: 0 | |
| [2026-06-26 20:13:07 DP1 TP1] Decode batch, #running-req: 32, #token: 1639360, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.01, #queue-req: 0 | |
| [2026-06-26 20:13:07 DP0 TP0] Decode batch, #running-req: 32, #token: 1636416, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1115.01, #queue-req: 0 | |
| [2026-06-26 20:13:08 DP2 TP2] Decode batch, #running-req: 32, #token: 1640832, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1111.34, #queue-req: 0 | |
| [2026-06-26 20:13:08 DP3 TP3] Decode batch, #running-req: 32, #token: 1639680, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1111.34, #queue-req: 0 | |
| [2026-06-26 20:13:08 DP1 TP1] Decode batch, #running-req: 32, #token: 1640960, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1111.35, #queue-req: 0 | |
| [2026-06-26 20:13:08 DP0 TP0] Decode batch, #running-req: 32, #token: 1637568, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1111.35, #queue-req: 0 | |
| [2026-06-26 20:13:09 DP2 TP2] Decode batch, #running-req: 32, #token: 1642240, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.24, #queue-req: 0 | |
| [2026-06-26 20:13:09 DP3 TP3] Decode batch, #running-req: 32, #token: 1640576, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.21, #queue-req: 0 | |
| [2026-06-26 20:13:09 DP1 TP1] Decode batch, #running-req: 32, #token: 1642368, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.21, #queue-req: 0 | |
| [2026-06-26 20:13:09 DP0 TP0] Decode batch, #running-req: 32, #token: 1639040, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.21, #queue-req: 0 | |
| [2026-06-26 20:13:11 DP2 TP2] Decode batch, #running-req: 32, #token: 1643328, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1109.43, #queue-req: 0 | |
| [2026-06-26 20:13:11 DP3 TP3] Decode batch, #running-req: 32, #token: 1642048, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1109.44, #queue-req: 0 | |
| [2026-06-26 20:13:11 DP1 TP1] Decode batch, #running-req: 32, #token: 1643328, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1109.44, #queue-req: 0 | |
| [2026-06-26 20:13:11 DP0 TP0] Decode batch, #running-req: 32, #token: 1640320, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1109.44, #queue-req: 0 | |
| [2026-06-26 20:13:12 DP2 TP2] Decode batch, #running-req: 32, #token: 1644736, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1110.15, #queue-req: 0 | |
| [2026-06-26 20:13:12 DP3 TP3] Decode batch, #running-req: 32, #token: 1643456, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1110.04, #queue-req: 0 | |
| [2026-06-26 20:13:12 DP0 TP0] Decode batch, #running-req: 32, #token: 1641536, token usage: 0.92, cuda graph: True, gen throughput (token/s): 1110.01, #queue-req: 0 | |
| [2026-06-26 20:13:12 DP1 TP1] Decode batch, #running-req: 32, #token: 1644800, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1110.01, #queue-req: 0 | |
| [2026-06-26 20:13:13 DP2 TP2] Decode batch, #running-req: 32, #token: 1645824, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.91, #queue-req: 0 | |
| [2026-06-26 20:13:13 DP3 TP3] Decode batch, #running-req: 32, #token: 1644544, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.90, #queue-req: 0 | |
| [2026-06-26 20:13:13 DP1 TP1] Decode batch, #running-req: 32, #token: 1645952, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.91, #queue-req: 0 | |
| [2026-06-26 20:13:13 DP0 TP0] Decode batch, #running-req: 32, #token: 1642944, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.91, #queue-req: 0 | |
| [2026-06-26 20:13:14 DP2 TP2] Decode batch, #running-req: 32, #token: 1647168, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.12, #queue-req: 0 | |
| [2026-06-26 20:13:14 DP3 TP3] Decode batch, #running-req: 32, #token: 1645952, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.15, #queue-req: 0 | |
| [2026-06-26 20:13:14 DP0 TP0] Decode batch, #running-req: 32, #token: 1644096, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.16, #queue-req: 0 | |
| [2026-06-26 20:13:14 DP1 TP1] Decode batch, #running-req: 32, #token: 1647232, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.16, #queue-req: 0 | |
| [2026-06-26 20:13:15 DP2 TP2] Decode batch, #running-req: 32, #token: 1648448, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.58, #queue-req: 0 | |
| [2026-06-26 20:13:15 DP3 TP3] Decode batch, #running-req: 32, #token: 1647360, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.58, #queue-req: 0 | |
| [2026-06-26 20:13:15 DP1 TP1] Decode batch, #running-req: 32, #token: 1648704, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.57, #queue-req: 0 | |
| [2026-06-26 20:13:15 DP0 TP0] Decode batch, #running-req: 32, #token: 1645504, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.57, #queue-req: 0 | |
| [2026-06-26 20:13:16 DP2 TP2] Decode batch, #running-req: 32, #token: 1649728, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.18, #queue-req: 0 | |
| [2026-06-26 20:13:16 DP3 TP3] Decode batch, #running-req: 32, #token: 1648256, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.16, #queue-req: 0 | |
| [2026-06-26 20:13:16 DP1 TP1] Decode batch, #running-req: 32, #token: 1649600, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.18, #queue-req: 0 | |
| [2026-06-26 20:13:16 DP0 TP0] Decode batch, #running-req: 32, #token: 1646656, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1108.18, #queue-req: 0 | |
| [2026-06-26 20:13:18 DP2 TP2] Decode batch, #running-req: 32, #token: 1651072, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.14, #queue-req: 0 | |
| [2026-06-26 20:13:18 DP3 TP3] Decode batch, #running-req: 32, #token: 1649920, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.24, #queue-req: 0 | |
| [2026-06-26 20:13:18 DP1 TP1] Decode batch, #running-req: 32, #token: 1651200, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.23, #queue-req: 0 | |
| [2026-06-26 20:13:18 DP0 TP0] Decode batch, #running-req: 32, #token: 1647808, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.24, #queue-req: 0 | |
| [2026-06-26 20:13:19 DP2 TP2] Decode batch, #running-req: 32, #token: 1652480, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1106.96, #queue-req: 0 | |
| [2026-06-26 20:13:19 DP3 TP3] Decode batch, #running-req: 32, #token: 1650816, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1106.79, #queue-req: 0 | |
| [2026-06-26 20:13:19 DP0 TP0] Decode batch, #running-req: 32, #token: 1649280, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1106.80, #queue-req: 0 | |
| [2026-06-26 20:13:19 DP1 TP1] Decode batch, #running-req: 32, #token: 1652608, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1106.80, #queue-req: 0 | |
| [2026-06-26 20:13:20 DP2 TP2] Decode batch, #running-req: 32, #token: 1653568, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.71, #queue-req: 0 | |
| [2026-06-26 20:13:20 DP0 TP0] Decode batch, #running-req: 32, #token: 1650560, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.71, #queue-req: 0 | |
| [2026-06-26 20:13:20 DP1 TP1] Decode batch, #running-req: 32, #token: 1653568, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.71, #queue-req: 0 | |
| [2026-06-26 20:13:20 DP3 TP3] Decode batch, #running-req: 32, #token: 1652288, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.70, #queue-req: 0 | |
| [2026-06-26 20:13:21 DP2 TP2] Decode batch, #running-req: 32, #token: 1654976, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.31, #queue-req: 0 | |
| [2026-06-26 20:13:21 DP3 TP3] Decode batch, #running-req: 32, #token: 1653696, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.35, #queue-req: 0 | |
| [2026-06-26 20:13:21 DP1 TP1] Decode batch, #running-req: 32, #token: 1655040, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.35, #queue-req: 0 | |
| [2026-06-26 20:13:21 DP0 TP0] Decode batch, #running-req: 32, #token: 1651776, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1105.35, #queue-req: 0 | |
| [2026-06-26 20:13:22 DP2 TP2] Decode batch, #running-req: 32, #token: 1656064, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.50, #queue-req: 0 | |
| [2026-06-26 20:13:22 DP3 TP3] Decode batch, #running-req: 32, #token: 1654784, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.49, #queue-req: 0 | |
| [2026-06-26 20:13:22 DP0 TP0] Decode batch, #running-req: 32, #token: 1653184, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.49, #queue-req: 0 | |
| [2026-06-26 20:13:22 DP1 TP1] Decode batch, #running-req: 32, #token: 1656192, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.49, #queue-req: 0 | |
| [2026-06-26 20:13:23 DP2 TP2] Decode batch, #running-req: 32, #token: 1657408, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.07, #queue-req: 0 | |
| [2026-06-26 20:13:23 DP3 TP3] Decode batch, #running-req: 32, #token: 1656192, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.14, #queue-req: 0 | |
| [2026-06-26 20:13:23 DP1 TP1] Decode batch, #running-req: 32, #token: 1657472, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.13, #queue-req: 0 | |
| [2026-06-26 20:13:23 DP0 TP0] Decode batch, #running-req: 32, #token: 1654336, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.13, #queue-req: 0 | |
| [2026-06-26 20:13:24 DP2 TP2] Decode batch, #running-req: 32, #token: 1658688, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.26, #queue-req: 0 | |
| [2026-06-26 20:13:25 DP0 TP0] Decode batch, #running-req: 32, #token: 1655744, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.21, #queue-req: 0 | |
| [2026-06-26 20:13:25 DP1 TP1] Decode batch, #running-req: 32, #token: 1658944, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.20, #queue-req: 0 | |
| [2026-06-26 20:13:25 DP3 TP3] Decode batch, #running-req: 32, #token: 1657600, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1104.16, #queue-req: 0 | |
| [2026-06-26 20:13:26 DP2 TP2] Decode batch, #running-req: 32, #token: 1659968, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.33, #queue-req: 0 | |
| [2026-06-26 20:13:26 DP3 TP3] Decode batch, #running-req: 32, #token: 1658496, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.25, #queue-req: 0 | |
| [2026-06-26 20:13:26 DP1 TP1] Decode batch, #running-req: 32, #token: 1659840, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.22, #queue-req: 0 | |
| [2026-06-26 20:13:26 DP0 TP0] Decode batch, #running-req: 32, #token: 1656896, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.21, #queue-req: 0 | |
| [2026-06-26 20:13:27 DP2 TP2] Decode batch, #running-req: 32, #token: 1661312, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.01, #queue-req: 0 | |
| [2026-06-26 20:13:27 DP3 TP3] Decode batch, #running-req: 32, #token: 1660160, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.18, #queue-req: 0 | |
| [2026-06-26 20:13:27 DP0 TP0] Decode batch, #running-req: 32, #token: 1658048, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1103.18, #queue-req: 0 | |
| [2026-06-26 20:13:27 DP1 TP1] Decode batch, #running-req: 32, #token: 1661440, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.18, #queue-req: 0 | |
| [2026-06-26 20:13:28 DP2 TP2] Decode batch, #running-req: 32, #token: 1662720, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.00, #queue-req: 0 | |
| [2026-06-26 20:13:28 DP3 TP3] Decode batch, #running-req: 32, #token: 1661056, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.76, #queue-req: 0 | |
| [2026-06-26 20:13:28 DP0 TP0] Decode batch, #running-req: 32, #token: 1659520, token usage: 0.93, cuda graph: True, gen throughput (token/s): 1102.76, #queue-req: 0 | |
| [2026-06-26 20:13:28 DP1 TP1] Decode batch, #running-req: 32, #token: 1662848, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.76, #queue-req: 0 | |
| [2026-06-26 20:13:29 DP2 TP2] Decode batch, #running-req: 32, #token: 1663808, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1100.92, #queue-req: 0 | |
| [2026-06-26 20:13:29 DP3 TP3] Decode batch, #running-req: 32, #token: 1662528, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.06, #queue-req: 0 | |
| [2026-06-26 20:13:29 DP1 TP1] Decode batch, #running-req: 32, #token: 1663808, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.06, #queue-req: 0 | |
| [2026-06-26 20:13:29 DP0 TP0] Decode batch, #running-req: 32, #token: 1660800, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.05, #queue-req: 0 | |
| [2026-06-26 20:13:30 DP2 TP2] Decode batch, #running-req: 32, #token: 1665216, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.35, #queue-req: 0 | |
| [2026-06-26 20:13:30 DP3 TP3] Decode batch, #running-req: 32, #token: 1663936, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.36, #queue-req: 0 | |
| [2026-06-26 20:13:30 DP1 TP1] Decode batch, #running-req: 32, #token: 1665280, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.36, #queue-req: 0 | |
| [2026-06-26 20:13:30 DP0 TP0] Decode batch, #running-req: 32, #token: 1662016, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.35, #queue-req: 0 | |
| [2026-06-26 20:13:31 DP2 TP2] Decode batch, #running-req: 32, #token: 1666304, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.83, #queue-req: 0 | |
| [2026-06-26 20:13:31 DP3 TP3] Decode batch, #running-req: 32, #token: 1665024, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.79, #queue-req: 0 | |
| [2026-06-26 20:13:31 DP1 TP1] Decode batch, #running-req: 32, #token: 1666432, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.80, #queue-req: 0 | |
| [2026-06-26 20:13:31 DP0 TP0] Decode batch, #running-req: 32, #token: 1663424, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.80, #queue-req: 0 | |
| [2026-06-26 20:13:33 DP2 TP2] Decode batch, #running-req: 32, #token: 1667648, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.66, #queue-req: 0 | |
| [2026-06-26 20:13:33 DP3 TP3] Decode batch, #running-req: 32, #token: 1666432, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.81, #queue-req: 0 | |
| [2026-06-26 20:13:33 DP0 TP0] Decode batch, #running-req: 32, #token: 1664576, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.80, #queue-req: 0 | |
| [2026-06-26 20:13:33 DP1 TP1] Decode batch, #running-req: 32, #token: 1667712, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.80, #queue-req: 0 | |
| [2026-06-26 20:13:34 DP2 TP2] Decode batch, #running-req: 32, #token: 1668928, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.45, #queue-req: 0 | |
| [2026-06-26 20:13:34 DP1 TP1] Decode batch, #running-req: 32, #token: 1669184, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.48, #queue-req: 0 | |
| [2026-06-26 20:13:34 DP0 TP0] Decode batch, #running-req: 32, #token: 1665984, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.49, #queue-req: 0 | |
| [2026-06-26 20:13:34 DP3 TP3] Decode batch, #running-req: 32, #token: 1667840, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.46, #queue-req: 0 | |
| [2026-06-26 20:13:35 DP2 TP2] Decode batch, #running-req: 32, #token: 1670208, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.79, #queue-req: 0 | |
| [2026-06-26 20:13:35 DP3 TP3] Decode batch, #running-req: 32, #token: 1668736, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.72, #queue-req: 0 | |
| [2026-06-26 20:13:35 DP1 TP1] Decode batch, #running-req: 32, #token: 1670080, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.70, #queue-req: 0 | |
| [2026-06-26 20:13:35 DP0 TP0] Decode batch, #running-req: 32, #token: 1667136, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1103.69, #queue-req: 0 | |
| [2026-06-26 20:13:36 DP2 TP2] Decode batch, #running-req: 32, #token: 1671552, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.80, #queue-req: 0 | |
| [2026-06-26 20:13:36 DP3 TP3] Decode batch, #running-req: 32, #token: 1670400, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.82, #queue-req: 0 | |
| [2026-06-26 20:13:36 DP1 TP1] Decode batch, #running-req: 32, #token: 1671680, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.83, #queue-req: 0 | |
| [2026-06-26 20:13:36 DP0 TP0] Decode batch, #running-req: 32, #token: 1668288, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1102.83, #queue-req: 0 | |
| [2026-06-26 20:13:37 DP2 TP2] Decode batch, #running-req: 32, #token: 1672960, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.73, #queue-req: 0 | |
| [2026-06-26 20:13:37 DP1 TP1] Decode batch, #running-req: 32, #token: 1673088, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.62, #queue-req: 0 | |
| [2026-06-26 20:13:37 DP0 TP0] Decode batch, #running-req: 32, #token: 1669760, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.61, #queue-req: 0 | |
| [2026-06-26 20:13:37 DP3 TP3] Decode batch, #running-req: 32, #token: 1671296, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1101.61, #queue-req: 0 | |
| [2026-06-26 20:13:38 DP2 TP2] Decode batch, #running-req: 32, #token: 1674048, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1100.23, #queue-req: 0 | |
| [2026-06-26 20:13:38 DP3 TP3] Decode batch, #running-req: 32, #token: 1672768, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1100.09, #queue-req: 0 | |
| [2026-06-26 20:13:38 DP1 TP1] Decode batch, #running-req: 32, #token: 1674048, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1100.07, #queue-req: 0 | |
| [2026-06-26 20:13:38 DP0 TP0] Decode batch, #running-req: 32, #token: 1671040, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1100.07, #queue-req: 0 | |
| [2026-06-26 20:13:40 DP2 TP2] Decode batch, #running-req: 32, #token: 1675456, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.17, #queue-req: 0 | |
| [2026-06-26 20:13:40 DP3 TP3] Decode batch, #running-req: 32, #token: 1674176, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.25, #queue-req: 0 | |
| [2026-06-26 20:13:40 DP1 TP1] Decode batch, #running-req: 32, #token: 1675520, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.27, #queue-req: 0 | |
| [2026-06-26 20:13:40 DP0 TP0] Decode batch, #running-req: 32, #token: 1672256, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.26, #queue-req: 0 | |
| [2026-06-26 20:13:41 DP2 TP2] Decode batch, #running-req: 32, #token: 1676544, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1098.35, #queue-req: 0 | |
| [2026-06-26 20:13:41 DP3 TP3] Decode batch, #running-req: 32, #token: 1675264, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1098.30, #queue-req: 0 | |
| [2026-06-26 20:13:41 DP1 TP1] Decode batch, #running-req: 32, #token: 1676672, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1098.29, #queue-req: 0 | |
| [2026-06-26 20:13:41 DP0 TP0] Decode batch, #running-req: 32, #token: 1673664, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1098.29, #queue-req: 0 | |
| [2026-06-26 20:13:42 DP2 TP2] Decode batch, #running-req: 32, #token: 1677888, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1096.75, #queue-req: 0 | |
| [2026-06-26 20:13:42 DP3 TP3] Decode batch, #running-req: 32, #token: 1676672, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1096.81, #queue-req: 0 | |
| [2026-06-26 20:13:42 DP1 TP1] Decode batch, #running-req: 32, #token: 1677952, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1096.83, #queue-req: 0 | |
| [2026-06-26 20:13:42 DP0 TP0] Decode batch, #running-req: 32, #token: 1674816, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1096.84, #queue-req: 0 | |
| [2026-06-26 20:13:43 DP2 TP2] Decode batch, #running-req: 32, #token: 1679168, token usage: 0.95, cuda graph: True, gen throughput (token/s): 1097.33, #queue-req: 0 | |
| [2026-06-26 20:13:43 DP3 TP3] Decode batch, #running-req: 32, #token: 1678080, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.40, #queue-req: 0 | |
| [2026-06-26 20:13:43 DP1 TP1] Decode batch, #running-req: 32, #token: 1679424, token usage: 0.95, cuda graph: True, gen throughput (token/s): 1097.37, #queue-req: 0 | |
| [2026-06-26 20:13:43 DP0 TP0] Decode batch, #running-req: 32, #token: 1676224, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1097.38, #queue-req: 0 | |
| [2026-06-26 20:13:44 DP2 TP2] Decode batch, #running-req: 32, #token: 1680448, token usage: 0.95, cuda graph: True, gen throughput (token/s): 1098.25, #queue-req: 0 | |
| [2026-06-26 20:13:44 DP3 TP3] Decode batch, #running-req: 32, #token: 1678976, token usage: 0.95, cuda graph: True, gen throughput (token/s): 1098.11, #queue-req: 0 | |
| [2026-06-26 20:13:44 DP1 TP1] Decode batch, #running-req: 32, #token: 1680320, token usage: 0.95, cuda graph: True, gen throughput (token/s): 1098.13, #queue-req: 0 | |
| [2026-06-26 20:13:44 DP0 TP0] Decode batch, #running-req: 32, #token: 1677376, token usage: 0.94, cuda graph: True, gen throughput (token/s): 1098.12, #queue-req: 0 | |
| [2026-06-26 20:13:45] INFO: 127.0.0.1:47994 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:45 DP2 TP2] Decode batch, #running-req: 32, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1071.31, #queue-req: 0 | |
| [2026-06-26 20:13:45 DP3 TP3] Decode batch, #running-req: 32, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1095.08, #queue-req: 0 | |
| [2026-06-26 20:13:45 DP1 TP1] Decode batch, #running-req: 32, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1071.08, #queue-req: 0 | |
| [2026-06-26 20:13:45 DP0 TP0] Decode batch, #running-req: 32, #token: 0, token usage: 0.00, cuda graph: True, gen throughput (token/s): 1071.08, #queue-req: 0 | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47648 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47784 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47812 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47852 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47900 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47940 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47992 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48020 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48052 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48090 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48116 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48140 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48174 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48204 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48490 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48240 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48286 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48314 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48354 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48388 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48440 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48540 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48584 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48610 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48636 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48660 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48706 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48724 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48750 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47766 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47682 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47718 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47754 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47786 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47824 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47860 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47914 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47956 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47998 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48022 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48058 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48094 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48118 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48150 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48176 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48210 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48250 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48298 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48324 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48370 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48394 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48446 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48498 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48554 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48594 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48614 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48638 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48670 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48708 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48736 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48764 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47694 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47664 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47742 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47774 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47806 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47846 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47886 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47932 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47980 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48008 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48044 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48086 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48108 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48138 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48164 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48198 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48228 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48280 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48310 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48350 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48382 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48424 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48476 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48524 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48576 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48598 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48630 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48654 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48720 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48746 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48780 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47704 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47668 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47732 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47772 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47794 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47832 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47870 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47918 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:47964 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48006 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48038 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48072 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48100 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48128 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48160 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48186 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48214 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48266 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48300 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48340 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48376 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48410 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48462 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48570 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48596 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48626 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48652 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48686 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48714 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48738 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48776 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46] INFO: 127.0.0.1:48514 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:13:46 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 50176, token usage: 0.03, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 150.65 | |
| [2026-06-26 20:13:54 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 5, #pending-token: 0, cuda graph: False, input throughput (token/s): 861.93 | |
| [2026-06-26 20:13:54 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 5, #pending-token: 0, cuda graph: False, input throughput (token/s): 873.20 | |
| [2026-06-26 20:13:57 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 50368, token usage: 0.10, #running-req: 1, #queue-req: 11, #pending-token: 238522, cuda graph: False, input throughput (token/s): 830.25 | |
| [2026-06-26 20:13:57 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 60736, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 14, #pending-token: 59365, cuda graph: False, input throughput (token/s): 855.95 | |
| [2026-06-26 20:13:58 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 60352, #cached-token: 0, token usage: 0.10, #running-req: 1, #queue-req: 13, #pending-token: 298333, cuda graph: False, input throughput (token/s): 14736.22 | |
| [2026-06-26 20:13:58 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.10, #running-req: 1, #queue-req: 13, #pending-token: 298119, cuda graph: False, input throughput (token/s): 14546.39 | |
| [2026-06-26 20:14:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9088, #cached-token: 50304, token usage: 0.10, #running-req: 1, #queue-req: 19, #pending-token: 833659, cuda graph: False, input throughput (token/s): 2002.76 | |
| [2026-06-26 20:14:03 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 50368, token usage: 0.13, #running-req: 2, #queue-req: 18, #pending-token: 655677, cuda graph: False, input throughput (token/s): 1601.07 | |
| [2026-06-26 20:14:03 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.13, #running-req: 2, #queue-req: 21, #pending-token: 774444, cuda graph: False, input throughput (token/s): 13070.44 | |
| [2026-06-26 20:14:03 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59904, #cached-token: 0, token usage: 0.13, #running-req: 2, #queue-req: 21, #pending-token: 773609, cuda graph: False, input throughput (token/s): 13077.25 | |
| [2026-06-26 20:14:08 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.17, #running-req: 3, #queue-req: 27, #pending-token: 1250566, cuda graph: False, input throughput (token/s): 11439.10 | |
| [2026-06-26 20:14:08 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 50368, token usage: 0.13, #running-req: 2, #queue-req: 27, #pending-token: 1131859, cuda graph: False, input throughput (token/s): 1411.05 | |
| [2026-06-26 20:14:08 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.17, #running-req: 3, #queue-req: 27, #pending-token: 1073383, cuda graph: False, input throughput (token/s): 11396.89 | |
| [2026-06-26 20:14:08 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.17, #running-req: 3, #queue-req: 27, #pending-token: 1251125, cuda graph: False, input throughput (token/s): 11396.28 | |
| [2026-06-26 20:14:12 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.17, #running-req: 3, #queue-req: 27, #pending-token: 1609126, cuda graph: False, input throughput (token/s): 17540.36 | |
| [2026-06-26 20:14:13 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.20, #running-req: 4, #queue-req: 26, #pending-token: 1608667, cuda graph: False, input throughput (token/s): 12767.08 | |
| [2026-06-26 20:14:13 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.20, #running-req: 4, #queue-req: 26, #pending-token: 1610153, cuda graph: False, input throughput (token/s): 12780.44 | |
| [2026-06-26 20:14:13 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59328, #cached-token: 0, token usage: 0.20, #running-req: 4, #queue-req: 26, #pending-token: 1608244, cuda graph: False, input throughput (token/s): 12703.48 | |
| [2026-06-26 20:14:18 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 60224, #cached-token: 0, token usage: 0.24, #running-req: 5, #queue-req: 25, #pending-token: 1548495, cuda graph: False, input throughput (token/s): 11488.26 | |
| [2026-06-26 20:14:18 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9280, #cached-token: 50368, token usage: 0.20, #running-req: 4, #queue-req: 26, #pending-token: 1609025, cuda graph: False, input throughput (token/s): 1425.24 | |
| [2026-06-26 20:14:18 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59840, #cached-token: 0, token usage: 0.23, #running-req: 5, #queue-req: 25, #pending-token: 1550374, cuda graph: False, input throughput (token/s): 11414.88 | |
| [2026-06-26 20:14:18 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.23, #running-req: 5, #queue-req: 25, #pending-token: 1548693, cuda graph: False, input throughput (token/s): 11378.23 | |
| [2026-06-26 20:14:23 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59328, #cached-token: 0, token usage: 0.27, #running-req: 6, #queue-req: 24, #pending-token: 1491100, cuda graph: False, input throughput (token/s): 11270.76 | |
| [2026-06-26 20:14:23 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.27, #running-req: 6, #queue-req: 24, #pending-token: 1489035, cuda graph: False, input throughput (token/s): 11296.35 | |
| [2026-06-26 20:14:23 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.24, #running-req: 5, #queue-req: 25, #pending-token: 1549633, cuda graph: False, input throughput (token/s): 11271.15 | |
| [2026-06-26 20:14:23 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.27, #running-req: 6, #queue-req: 24, #pending-token: 1488934, cuda graph: False, input throughput (token/s): 11356.88 | |
| [2026-06-26 20:14:27 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.27, #running-req: 6, #queue-req: 24, #pending-token: 1490072, cuda graph: False, input throughput (token/s): 17672.99 | |
| [2026-06-26 20:14:28 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.30, #running-req: 7, #queue-req: 23, #pending-token: 1431497, cuda graph: False, input throughput (token/s): 12862.56 | |
| [2026-06-26 20:14:28 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.30, #running-req: 7, #queue-req: 23, #pending-token: 1429336, cuda graph: False, input throughput (token/s): 12876.93 | |
| [2026-06-26 20:14:28 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.30, #running-req: 7, #queue-req: 23, #pending-token: 1429556, cuda graph: False, input throughput (token/s): 12800.71 | |
| [2026-06-26 20:14:31 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.34, #running-req: 8, #queue-req: 22, #pending-token: 1371810, cuda graph: False, input throughput (token/s): 18141.12 | |
| [2026-06-26 20:14:33 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.34, #running-req: 8, #queue-req: 22, #pending-token: 1369989, cuda graph: False, input throughput (token/s): 12913.08 | |
| [2026-06-26 20:14:33 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.34, #running-req: 8, #queue-req: 22, #pending-token: 1370036, cuda graph: False, input throughput (token/s): 12936.07 | |
| [2026-06-26 20:14:33 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 50880, token usage: 0.30, #running-req: 7, #queue-req: 23, #pending-token: 1429867, cuda graph: False, input throughput (token/s): 1591.95 | |
| [2026-06-26 20:14:36 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.37, #running-req: 9, #queue-req: 21, #pending-token: 1310533, cuda graph: False, input throughput (token/s): 17989.49 | |
| [2026-06-26 20:14:37 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.37, #running-req: 9, #queue-req: 21, #pending-token: 1310598, cuda graph: False, input throughput (token/s): 12895.78 | |
| [2026-06-26 20:14:37 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.34, #running-req: 8, #queue-req: 22, #pending-token: 1370426, cuda graph: False, input throughput (token/s): 12881.13 | |
| [2026-06-26 20:14:37 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 50304, token usage: 0.37, #running-req: 9, #queue-req: 21, #pending-token: 1312407, cuda graph: False, input throughput (token/s): 1543.06 | |
| [2026-06-26 20:14:42 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 60096, #cached-token: 0, token usage: 0.40, #running-req: 10, #queue-req: 20, #pending-token: 1252317, cuda graph: False, input throughput (token/s): 11730.95 | |
| [2026-06-26 20:14:42 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 50496, token usage: 0.40, #running-req: 10, #queue-req: 20, #pending-token: 1250867, cuda graph: False, input throughput (token/s): 1430.83 | |
| [2026-06-26 20:14:42 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.37, #running-req: 9, #queue-req: 21, #pending-token: 1310655, cuda graph: False, input throughput (token/s): 11664.40 | |
| [2026-06-26 20:14:42 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.40, #running-req: 10, #queue-req: 20, #pending-token: 1251058, cuda graph: False, input throughput (token/s): 11614.78 | |
| [2026-06-26 20:14:48 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.44, #running-req: 11, #queue-req: 19, #pending-token: 1191216, cuda graph: False, input throughput (token/s): 11517.40 | |
| [2026-06-26 20:14:48 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.44, #running-req: 11, #queue-req: 19, #pending-token: 1192712, cuda graph: False, input throughput (token/s): 11505.13 | |
| [2026-06-26 20:14:48 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.40, #running-req: 10, #queue-req: 20, #pending-token: 1251022, cuda graph: False, input throughput (token/s): 11505.49 | |
| [2026-06-26 20:14:48 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.44, #running-req: 11, #queue-req: 19, #pending-token: 1191627, cuda graph: False, input throughput (token/s): 11467.68 | |
| [2026-06-26 20:14:50 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.47, #running-req: 12, #queue-req: 18, #pending-token: 1131731, cuda graph: False, input throughput (token/s): 20847.31 | |
| [2026-06-26 20:14:50 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.47, #running-req: 12, #queue-req: 18, #pending-token: 1132176, cuda graph: False, input throughput (token/s): 20825.34 | |
| [2026-06-26 20:14:52 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.47, #running-req: 12, #queue-req: 18, #pending-token: 1133138, cuda graph: False, input throughput (token/s): 14675.48 | |
| [2026-06-26 20:14:52 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59328, #cached-token: 0, token usage: 0.44, #running-req: 11, #queue-req: 19, #pending-token: 1191699, cuda graph: False, input throughput (token/s): 14606.74 | |
| [2026-06-26 20:14:54 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 50304, token usage: 0.50, #running-req: 13, #queue-req: 17, #pending-token: 1072286, cuda graph: False, input throughput (token/s): 2288.08 | |
| [2026-06-26 20:14:54 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9152, #cached-token: 50304, token usage: 0.50, #running-req: 13, #queue-req: 17, #pending-token: 1072746, cuda graph: False, input throughput (token/s): 2288.04 | |
| [2026-06-26 20:14:56 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.50, #running-req: 13, #queue-req: 17, #pending-token: 1073573, cuda graph: False, input throughput (token/s): 14900.18 | |
| [2026-06-26 20:14:56 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.47, #running-req: 12, #queue-req: 18, #pending-token: 1132367, cuda graph: False, input throughput (token/s): 14840.26 | |
| [2026-06-26 20:15:01 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.54, #running-req: 14, #queue-req: 16, #pending-token: 1014027, cuda graph: False, input throughput (token/s): 11446.90 | |
| [2026-06-26 20:15:01 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 50432, token usage: 0.54, #running-req: 14, #queue-req: 16, #pending-token: 1012697, cuda graph: False, input throughput (token/s): 1437.88 | |
| [2026-06-26 20:15:01 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.50, #running-req: 13, #queue-req: 17, #pending-token: 1072994, cuda graph: False, input throughput (token/s): 11420.29 | |
| [2026-06-26 20:15:01 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 50304, token usage: 0.54, #running-req: 14, #queue-req: 16, #pending-token: 1013251, cuda graph: False, input throughput (token/s): 1437.89 | |
| [2026-06-26 20:15:06 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.57, #running-req: 15, #queue-req: 15, #pending-token: 954433, cuda graph: False, input throughput (token/s): 11469.78 | |
| [2026-06-26 20:15:06 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.54, #running-req: 14, #queue-req: 16, #pending-token: 1013235, cuda graph: False, input throughput (token/s): 11483.08 | |
| [2026-06-26 20:15:06 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.57, #running-req: 15, #queue-req: 15, #pending-token: 953561, cuda graph: False, input throughput (token/s): 11470.38 | |
| [2026-06-26 20:15:06 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.57, #running-req: 15, #queue-req: 15, #pending-token: 953182, cuda graph: False, input throughput (token/s): 11432.68 | |
| [2026-06-26 20:15:09 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.60, #running-req: 16, #queue-req: 14, #pending-token: 893541, cuda graph: False, input throughput (token/s): 17691.91 | |
| [2026-06-26 20:15:11 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.57, #running-req: 15, #queue-req: 15, #pending-token: 953894, cuda graph: False, input throughput (token/s): 12824.30 | |
| [2026-06-26 20:15:11 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 60032, #cached-token: 0, token usage: 0.60, #running-req: 16, #queue-req: 14, #pending-token: 893580, cuda graph: False, input throughput (token/s): 12948.71 | |
| [2026-06-26 20:15:11 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.60, #running-req: 16, #queue-req: 14, #pending-token: 894945, cuda graph: False, input throughput (token/s): 12820.68 | |
| [2026-06-26 20:15:16 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.64, #running-req: 17, #queue-req: 13, #pending-token: 835472, cuda graph: False, input throughput (token/s): 11350.37 | |
| [2026-06-26 20:15:16 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.60, #running-req: 16, #queue-req: 14, #pending-token: 894506, cuda graph: False, input throughput (token/s): 11301.28 | |
| [2026-06-26 20:15:16 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.64, #running-req: 17, #queue-req: 13, #pending-token: 833963, cuda graph: False, input throughput (token/s): 11361.42 | |
| [2026-06-26 20:15:16 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9344, #cached-token: 50496, token usage: 0.64, #running-req: 17, #queue-req: 13, #pending-token: 833751, cuda graph: False, input throughput (token/s): 1434.10 | |
| [2026-06-26 20:15:19 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.67, #running-req: 18, #queue-req: 12, #pending-token: 774118, cuda graph: False, input throughput (token/s): 17727.92 | |
| [2026-06-26 20:15:21 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.67, #running-req: 18, #queue-req: 12, #pending-token: 776068, cuda graph: False, input throughput (token/s): 12849.57 | |
| [2026-06-26 20:15:21 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 60032, #cached-token: 0, token usage: 0.64, #running-req: 17, #queue-req: 13, #pending-token: 834523, cuda graph: False, input throughput (token/s): 12976.77 | |
| [2026-06-26 20:15:21 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 60096, #cached-token: 0, token usage: 0.67, #running-req: 18, #queue-req: 12, #pending-token: 773874, cuda graph: False, input throughput (token/s): 12988.61 | |
| [2026-06-26 20:15:26 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.71, #running-req: 19, #queue-req: 11, #pending-token: 716644, cuda graph: False, input throughput (token/s): 11461.56 | |
| [2026-06-26 20:15:26 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 50560, token usage: 0.71, #running-req: 19, #queue-req: 11, #pending-token: 714369, cuda graph: False, input throughput (token/s): 1430.08 | |
| [2026-06-26 20:15:26 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.67, #running-req: 18, #queue-req: 12, #pending-token: 774762, cuda graph: False, input throughput (token/s): 11530.68 | |
| [2026-06-26 20:15:26 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.70, #running-req: 19, #queue-req: 11, #pending-token: 714531, cuda graph: False, input throughput (token/s): 11458.18 | |
| [2026-06-26 20:15:31 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.71, #running-req: 19, #queue-req: 11, #pending-token: 715204, cuda graph: False, input throughput (token/s): 11476.27 | |
| [2026-06-26 20:15:31 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 60352, #cached-token: 0, token usage: 0.74, #running-req: 20, #queue-req: 10, #pending-token: 656305, cuda graph: False, input throughput (token/s): 11613.94 | |
| [2026-06-26 20:15:31 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.74, #running-req: 20, #queue-req: 10, #pending-token: 654680, cuda graph: False, input throughput (token/s): 11492.12 | |
| [2026-06-26 20:15:31 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.74, #running-req: 20, #queue-req: 10, #pending-token: 655085, cuda graph: False, input throughput (token/s): 11440.45 | |
| [2026-06-26 20:15:36 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.77, #running-req: 21, #queue-req: 9, #pending-token: 595194, cuda graph: False, input throughput (token/s): 11495.13 | |
| [2026-06-26 20:15:36 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.74, #running-req: 20, #queue-req: 10, #pending-token: 655620, cuda graph: False, input throughput (token/s): 11488.01 | |
| [2026-06-26 20:15:36 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59968, #cached-token: 0, token usage: 0.77, #running-req: 21, #queue-req: 9, #pending-token: 596339, cuda graph: False, input throughput (token/s): 11568.92 | |
| [2026-06-26 20:15:36 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.77, #running-req: 21, #queue-req: 9, #pending-token: 595624, cuda graph: False, input throughput (token/s): 11485.99 | |
| [2026-06-26 20:15:41 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59840, #cached-token: 0, token usage: 0.81, #running-req: 22, #queue-req: 8, #pending-token: 536549, cuda graph: False, input throughput (token/s): 11559.41 | |
| [2026-06-26 20:15:41 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.77, #running-req: 21, #queue-req: 9, #pending-token: 596198, cuda graph: False, input throughput (token/s): 11483.92 | |
| [2026-06-26 20:15:41 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59328, #cached-token: 0, token usage: 0.81, #running-req: 22, #queue-req: 8, #pending-token: 535906, cuda graph: False, input throughput (token/s): 11448.15 | |
| [2026-06-26 20:15:41 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.81, #running-req: 22, #queue-req: 8, #pending-token: 536159, cuda graph: False, input throughput (token/s): 11494.07 | |
| [2026-06-26 20:15:47 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.84, #running-req: 23, #queue-req: 7, #pending-token: 476517, cuda graph: False, input throughput (token/s): 11418.10 | |
| [2026-06-26 20:15:47 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.84, #running-req: 23, #queue-req: 7, #pending-token: 477111, cuda graph: False, input throughput (token/s): 11419.26 | |
| [2026-06-26 20:15:47 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.81, #running-req: 22, #queue-req: 8, #pending-token: 536804, cuda graph: False, input throughput (token/s): 11418.99 | |
| [2026-06-26 20:15:47 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59392, #cached-token: 0, token usage: 0.84, #running-req: 23, #queue-req: 7, #pending-token: 476794, cuda graph: False, input throughput (token/s): 11409.97 | |
| [2026-06-26 20:15:50 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59904, #cached-token: 0, token usage: 0.84, #running-req: 23, #queue-req: 7, #pending-token: 476949, cuda graph: False, input throughput (token/s): 18025.74 | |
| [2026-06-26 20:15:51 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.87, #running-req: 24, #queue-req: 6, #pending-token: 417714, cuda graph: False, input throughput (token/s): 12830.78 | |
| [2026-06-26 20:15:51 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.87, #running-req: 24, #queue-req: 6, #pending-token: 416945, cuda graph: False, input throughput (token/s): 12843.18 | |
| [2026-06-26 20:15:51 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59840, #cached-token: 0, token usage: 0.87, #running-req: 24, #queue-req: 6, #pending-token: 416981, cuda graph: False, input throughput (token/s): 12911.11 | |
| [2026-06-26 20:15:56 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.91, #running-req: 25, #queue-req: 5, #pending-token: 357502, cuda graph: False, input throughput (token/s): 11526.12 | |
| [2026-06-26 20:15:56 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.91, #running-req: 25, #queue-req: 5, #pending-token: 358121, cuda graph: False, input throughput (token/s): 11550.30 | |
| [2026-06-26 20:15:56 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 9216, #cached-token: 50368, token usage: 0.87, #running-req: 24, #queue-req: 6, #pending-token: 417399, cuda graph: False, input throughput (token/s): 1423.34 | |
| [2026-06-26 20:15:56 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.91, #running-req: 25, #queue-req: 5, #pending-token: 357513, cuda graph: False, input throughput (token/s): 11527.39 | |
| [2026-06-26 20:16:02 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.94, #running-req: 26, #queue-req: 4, #pending-token: 297875, cuda graph: False, input throughput (token/s): 11308.55 | |
| [2026-06-26 20:16:02 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.94, #running-req: 26, #queue-req: 4, #pending-token: 298018, cuda graph: False, input throughput (token/s): 11265.04 | |
| [2026-06-26 20:16:02 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.94, #running-req: 26, #queue-req: 4, #pending-token: 298423, cuda graph: False, input throughput (token/s): 11309.56 | |
| [2026-06-26 20:16:02 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.91, #running-req: 25, #queue-req: 5, #pending-token: 357953, cuda graph: False, input throughput (token/s): 11261.95 | |
| [2026-06-26 20:16:07 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59968, #cached-token: 0, token usage: 0.97, #running-req: 27, #queue-req: 3, #pending-token: 238513, cuda graph: False, input throughput (token/s): 10520.23 | |
| [2026-06-26 20:16:07 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59328, #cached-token: 0, token usage: 0.97, #running-req: 27, #queue-req: 3, #pending-token: 238571, cuda graph: False, input throughput (token/s): 10398.91 | |
| [2026-06-26 20:16:07 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.97, #running-req: 27, #queue-req: 3, #pending-token: 238276, cuda graph: False, input throughput (token/s): 10482.80 | |
| [2026-06-26 20:16:07 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 60416, #cached-token: 0, token usage: 0.94, #running-req: 26, #queue-req: 4, #pending-token: 297596, cuda graph: False, input throughput (token/s): 10597.01 | |
| [2026-06-26 20:16:10 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.97, #running-req: 28, #queue-req: 3, #pending-token: 179008, cuda graph: False, input throughput (token/s): 26813.28 | |
| [2026-06-26 20:16:10 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.97, #running-req: 28, #queue-req: 3, #pending-token: 178618, cuda graph: False, input throughput (token/s): 26907.71 | |
| [2026-06-26 20:16:10 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59648, #cached-token: 0, token usage: 0.97, #running-req: 28, #queue-req: 3, #pending-token: 178954, cuda graph: False, input throughput (token/s): 26868.42 | |
| [2026-06-26 20:16:10 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.97, #running-req: 27, #queue-req: 3, #pending-token: 238085, cuda graph: False, input throughput (token/s): 19348.89 | |
| [2026-06-26 20:16:11 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.97, #running-req: 28, #queue-req: 3, #pending-token: 178341, cuda graph: False, input throughput (token/s): 498524.83 | |
| [2026-06-26 20:16:12 DP2 TP2] Decode batch, #running-req: 29, #token: 1730688, token usage: 0.97, cuda graph: True, gen throughput (token/s): 7.57, #queue-req: 3 | |
| [2026-06-26 20:16:12 DP0 TP0] Decode batch, #running-req: 29, #token: 1729344, token usage: 0.97, cuda graph: True, gen throughput (token/s): 7.96, #queue-req: 3 | |
| [2026-06-26 20:16:12 DP3 TP3] Decode batch, #running-req: 29, #token: 1730752, token usage: 0.97, cuda graph: True, gen throughput (token/s): 7.94, #queue-req: 3 | |
| [2026-06-26 20:16:12 DP1 TP1] Decode batch, #running-req: 29, #token: 1731456, token usage: 0.98, cuda graph: True, gen throughput (token/s): 7.96, #queue-req: 3 | |
| [2026-06-26 20:16:13 DP2 TP2] Decode batch, #running-req: 29, #token: 1731968, token usage: 0.98, cuda graph: True, gen throughput (token/s): 1001.22, #queue-req: 3 | |
| [2026-06-26 20:16:13 DP0 TP0] Decode batch, #running-req: 29, #token: 1730176, token usage: 0.97, cuda graph: True, gen throughput (token/s): 1001.01, #queue-req: 3 | |
| [2026-06-26 20:16:13 DP3 TP3] Decode batch, #running-req: 29, #token: 1731968, token usage: 0.98, cuda graph: True, gen throughput (token/s): 1000.96, #queue-req: 3 | |
| [2026-06-26 20:16:13 DP1 TP1] Decode batch, #running-req: 29, #token: 1732608, token usage: 0.98, cuda graph: True, gen throughput (token/s): 1001.00, #queue-req: 3 | |
| [2026-06-26 20:16:14 DP2 TP2] Decode batch, #running-req: 29, #token: 1732992, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.90, #queue-req: 3 | |
| [2026-06-26 20:16:14 DP0 TP0] Decode batch, #running-req: 29, #token: 1731520, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.87, #queue-req: 3 | |
| [2026-06-26 20:16:14 DP3 TP3] Decode batch, #running-req: 29, #token: 1732992, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.73, #queue-req: 3 | |
| [2026-06-26 20:16:14 DP1 TP1] Decode batch, #running-req: 29, #token: 1733568, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.73, #queue-req: 3 | |
| [2026-06-26 20:16:15 DP2 TP2] Decode batch, #running-req: 29, #token: 1734272, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.42, #queue-req: 3 | |
| [2026-06-26 20:16:15 DP0 TP0] Decode batch, #running-req: 29, #token: 1732992, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.23, #queue-req: 3 | |
| [2026-06-26 20:16:15 DP3 TP3] Decode batch, #running-req: 29, #token: 1734080, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.29, #queue-req: 3 | |
| [2026-06-26 20:16:15 DP1 TP1] Decode batch, #running-req: 29, #token: 1734976, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.28, #queue-req: 3 | |
| [2026-06-26 20:16:16 DP2 TP2] Decode batch, #running-req: 29, #token: 1735488, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.04, #queue-req: 3 | |
| [2026-06-26 20:16:16 DP0 TP0] Decode batch, #running-req: 29, #token: 1733696, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.86, #queue-req: 3 | |
| [2026-06-26 20:16:16 DP3 TP3] Decode batch, #running-req: 29, #token: 1735296, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.99, #queue-req: 3 | |
| [2026-06-26 20:16:16 DP1 TP1] Decode batch, #running-req: 29, #token: 1736000, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.79, #queue-req: 3 | |
| [2026-06-26 20:16:17 DP2 TP2] Decode batch, #running-req: 29, #token: 1736448, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.31, #queue-req: 3 | |
| [2026-06-26 20:16:17 DP0 TP0] Decode batch, #running-req: 29, #token: 1734976, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.59, #queue-req: 3 | |
| [2026-06-26 20:16:17 DP3 TP3] Decode batch, #running-req: 29, #token: 1736448, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.46, #queue-req: 3 | |
| [2026-06-26 20:16:17 DP1 TP1] Decode batch, #running-req: 29, #token: 1737024, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.68, #queue-req: 3 | |
| [2026-06-26 20:16:19 DP2 TP2] Decode batch, #running-req: 29, #token: 1737856, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.95, #queue-req: 3 | |
| [2026-06-26 20:16:19 DP0 TP0] Decode batch, #running-req: 29, #token: 1736064, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.88, #queue-req: 3 | |
| [2026-06-26 20:16:19 DP3 TP3] Decode batch, #running-req: 29, #token: 1737600, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.69, #queue-req: 3 | |
| [2026-06-26 20:16:19 DP1 TP1] Decode batch, #running-req: 29, #token: 1738496, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.68, #queue-req: 3 | |
| [2026-06-26 20:16:20 DP2 TP2] Decode batch, #running-req: 29, #token: 1739072, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.44, #queue-req: 3 | |
| [2026-06-26 20:16:20 DP0 TP0] Decode batch, #running-req: 29, #token: 1737152, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.37, #queue-req: 3 | |
| [2026-06-26 20:16:20 DP3 TP3] Decode batch, #running-req: 29, #token: 1738816, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.46, #queue-req: 3 | |
| [2026-06-26 20:16:20 DP1 TP1] Decode batch, #running-req: 29, #token: 1739328, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.46, #queue-req: 3 | |
| [2026-06-26 20:16:21 DP2 TP2] Decode batch, #running-req: 29, #token: 1739968, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.89, #queue-req: 3 | |
| [2026-06-26 20:16:21 DP0 TP0] Decode batch, #running-req: 29, #token: 1738624, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.65, #queue-req: 3 | |
| [2026-06-26 20:16:21 DP1 TP1] Decode batch, #running-req: 29, #token: 1740736, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.60, #queue-req: 3 | |
| [2026-06-26 20:16:21 DP3 TP3] Decode batch, #running-req: 29, #token: 1740032, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.59, #queue-req: 3 | |
| [2026-06-26 20:16:22 DP2 TP2] Decode batch, #running-req: 29, #token: 1741248, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.01, #queue-req: 3 | |
| [2026-06-26 20:16:22 DP0 TP0] Decode batch, #running-req: 29, #token: 1739456, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.17, #queue-req: 3 | |
| [2026-06-26 20:16:22 DP3 TP3] Decode batch, #running-req: 29, #token: 1741248, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.55, #queue-req: 3 | |
| [2026-06-26 20:16:22 DP1 TP1] Decode batch, #running-req: 29, #token: 1741888, token usage: 0.98, cuda graph: True, gen throughput (token/s): 995.31, #queue-req: 3 | |
| [2026-06-26 20:16:23 DP2 TP2] Decode batch, #running-req: 29, #token: 1742272, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.78, #queue-req: 3 | |
| [2026-06-26 20:16:23 DP0 TP0] Decode batch, #running-req: 29, #token: 1740800, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.67, #queue-req: 3 | |
| [2026-06-26 20:16:23 DP3 TP3] Decode batch, #running-req: 29, #token: 1742272, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.26, #queue-req: 3 | |
| [2026-06-26 20:16:23 DP1 TP1] Decode batch, #running-req: 29, #token: 1742848, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.48, #queue-req: 3 | |
| [2026-06-26 20:16:24 DP2 TP2] Decode batch, #running-req: 29, #token: 1743552, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.14, #queue-req: 3 | |
| [2026-06-26 20:16:24 DP0 TP0] Decode batch, #running-req: 29, #token: 1742272, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.40, #queue-req: 3 | |
| [2026-06-26 20:16:24 DP3 TP3] Decode batch, #running-req: 29, #token: 1743360, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.67, #queue-req: 3 | |
| [2026-06-26 20:16:24 DP1 TP1] Decode batch, #running-req: 29, #token: 1744256, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.64, #queue-req: 3 | |
| [2026-06-26 20:16:26 DP2 TP2] Decode batch, #running-req: 29, #token: 1744768, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.45, #queue-req: 3 | |
| [2026-06-26 20:16:26 DP0 TP0] Decode batch, #running-req: 29, #token: 1742976, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.31, #queue-req: 3 | |
| [2026-06-26 20:16:26 DP3 TP3] Decode batch, #running-req: 29, #token: 1744576, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.17, #queue-req: 3 | |
| [2026-06-26 20:16:26 DP1 TP1] Decode batch, #running-req: 29, #token: 1745280, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.20, #queue-req: 3 | |
| [2026-06-26 20:16:27 DP2 TP2] Decode batch, #running-req: 29, #token: 1745728, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.81, #queue-req: 3 | |
| [2026-06-26 20:16:27 DP0 TP0] Decode batch, #running-req: 29, #token: 1744256, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.85, #queue-req: 3 | |
| [2026-06-26 20:16:27 DP3 TP3] Decode batch, #running-req: 29, #token: 1745728, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.79, #queue-req: 3 | |
| [2026-06-26 20:16:27 DP1 TP1] Decode batch, #running-req: 29, #token: 1746304, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.78, #queue-req: 3 | |
| [2026-06-26 20:16:28 DP2 TP2] Decode batch, #running-req: 29, #token: 1747136, token usage: 0.98, cuda graph: True, gen throughput (token/s): 997.60, #queue-req: 3 | |
| [2026-06-26 20:16:28 DP0 TP0] Decode batch, #running-req: 29, #token: 1745344, token usage: 0.98, cuda graph: True, gen throughput (token/s): 997.77, #queue-req: 3 | |
| [2026-06-26 20:16:28 DP3 TP3] Decode batch, #running-req: 29, #token: 1746880, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.12, #queue-req: 3 | |
| [2026-06-26 20:16:28 DP1 TP1] Decode batch, #running-req: 29, #token: 1747776, token usage: 0.98, cuda graph: True, gen throughput (token/s): 997.89, #queue-req: 3 | |
| [2026-06-26 20:16:29 DP2 TP2] Decode batch, #running-req: 29, #token: 1748352, token usage: 0.98, cuda graph: True, gen throughput (token/s): 999.22, #queue-req: 3 | |
| [2026-06-26 20:16:29 DP0 TP0] Decode batch, #running-req: 29, #token: 1746432, token usage: 0.98, cuda graph: True, gen throughput (token/s): 999.08, #queue-req: 3 | |
| [2026-06-26 20:16:29 DP3 TP3] Decode batch, #running-req: 29, #token: 1748096, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.79, #queue-req: 3 | |
| [2026-06-26 20:16:29 DP1 TP1] Decode batch, #running-req: 29, #token: 1748608, token usage: 0.98, cuda graph: True, gen throughput (token/s): 998.99, #queue-req: 3 | |
| [2026-06-26 20:16:30 DP2 TP2] Decode batch, #running-req: 29, #token: 1749248, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.94, #queue-req: 3 | |
| [2026-06-26 20:16:30 DP0 TP0] Decode batch, #running-req: 29, #token: 1747904, token usage: 0.98, cuda graph: True, gen throughput (token/s): 996.97, #queue-req: 3 | |
| [2026-06-26 20:16:30 DP3 TP3] Decode batch, #running-req: 29, #token: 1749312, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.98, #queue-req: 3 | |
| [2026-06-26 20:16:30 DP1 TP1] Decode batch, #running-req: 29, #token: 1750016, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.97, #queue-req: 3 | |
| [2026-06-26 20:16:31 DP2 TP2] Decode batch, #running-req: 29, #token: 1750528, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.34, #queue-req: 3 | |
| [2026-06-26 20:16:31 DP0 TP0] Decode batch, #running-req: 29, #token: 1748736, token usage: 0.98, cuda graph: True, gen throughput (token/s): 994.31, #queue-req: 3 | |
| [2026-06-26 20:16:31 DP3 TP3] Decode batch, #running-req: 29, #token: 1750528, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.20, #queue-req: 3 | |
| [2026-06-26 20:16:31 DP1 TP1] Decode batch, #running-req: 29, #token: 1751168, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.25, #queue-req: 3 | |
| [2026-06-26 20:16:32 DP2 TP2] Decode batch, #running-req: 29, #token: 1751552, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.55, #queue-req: 3 | |
| [2026-06-26 20:16:33 DP0 TP0] Decode batch, #running-req: 29, #token: 1750080, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.64, #queue-req: 3 | |
| [2026-06-26 20:16:33 DP3 TP3] Decode batch, #running-req: 29, #token: 1751552, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.79, #queue-req: 3 | |
| [2026-06-26 20:16:33 DP1 TP1] Decode batch, #running-req: 29, #token: 1752128, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.78, #queue-req: 3 | |
| [2026-06-26 20:16:34 DP2 TP2] Decode batch, #running-req: 29, #token: 1752832, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.09, #queue-req: 3 | |
| [2026-06-26 20:16:34 DP0 TP0] Decode batch, #running-req: 29, #token: 1751552, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.85, #queue-req: 3 | |
| [2026-06-26 20:16:34 DP3 TP3] Decode batch, #running-req: 29, #token: 1752640, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.02, #queue-req: 3 | |
| [2026-06-26 20:16:34 DP1 TP1] Decode batch, #running-req: 29, #token: 1753536, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.81, #queue-req: 3 | |
| [2026-06-26 20:16:35 DP2 TP2] Decode batch, #running-req: 29, #token: 1754048, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.17, #queue-req: 3 | |
| [2026-06-26 20:16:35 DP0 TP0] Decode batch, #running-req: 29, #token: 1752256, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.21, #queue-req: 3 | |
| [2026-06-26 20:16:35 DP3 TP3] Decode batch, #running-req: 29, #token: 1753856, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.99, #queue-req: 3 | |
| [2026-06-26 20:16:35 DP1 TP1] Decode batch, #running-req: 29, #token: 1754560, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.21, #queue-req: 3 | |
| [2026-06-26 20:16:36 DP2 TP2] Decode batch, #running-req: 29, #token: 1755008, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.55, #queue-req: 3 | |
| [2026-06-26 20:16:36 DP0 TP0] Decode batch, #running-req: 29, #token: 1753536, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.84, #queue-req: 3 | |
| [2026-06-26 20:16:36 DP3 TP3] Decode batch, #running-req: 29, #token: 1755008, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.83, #queue-req: 3 | |
| [2026-06-26 20:16:36 DP1 TP1] Decode batch, #running-req: 29, #token: 1755584, token usage: 0.99, cuda graph: True, gen throughput (token/s): 995.83, #queue-req: 3 | |
| [2026-06-26 20:16:37 DP2 TP2] Decode batch, #running-req: 29, #token: 1756416, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.48, #queue-req: 3 | |
| [2026-06-26 20:16:37 DP0 TP0] Decode batch, #running-req: 29, #token: 1754624, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.43, #queue-req: 3 | |
| [2026-06-26 20:16:37 DP3 TP3] Decode batch, #running-req: 29, #token: 1756160, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.55, #queue-req: 3 | |
| [2026-06-26 20:16:37 DP1 TP1] Decode batch, #running-req: 29, #token: 1757056, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.55, #queue-req: 3 | |
| [2026-06-26 20:16:38 DP2 TP2] Decode batch, #running-req: 29, #token: 1757632, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.71, #queue-req: 3 | |
| [2026-06-26 20:16:38 DP0 TP0] Decode batch, #running-req: 29, #token: 1755712, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.71, #queue-req: 3 | |
| [2026-06-26 20:16:38 DP3 TP3] Decode batch, #running-req: 29, #token: 1757376, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.54, #queue-req: 3 | |
| [2026-06-26 20:16:38 DP1 TP1] Decode batch, #running-req: 29, #token: 1757888, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.54, #queue-req: 3 | |
| [2026-06-26 20:16:39 DP2 TP2] Decode batch, #running-req: 29, #token: 1758528, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.42, #queue-req: 3 | |
| [2026-06-26 20:16:40 DP0 TP0] Decode batch, #running-req: 29, #token: 1757184, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.26, #queue-req: 3 | |
| [2026-06-26 20:16:40 DP3 TP3] Decode batch, #running-req: 29, #token: 1758592, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.42, #queue-req: 3 | |
| [2026-06-26 20:16:40 DP1 TP1] Decode batch, #running-req: 29, #token: 1759296, token usage: 0.99, cuda graph: True, gen throughput (token/s): 997.19, #queue-req: 3 | |
| [2026-06-26 20:16:41 DP2 TP2] Decode batch, #running-req: 29, #token: 1759808, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.59, #queue-req: 3 | |
| [2026-06-26 20:16:41 DP0 TP0] Decode batch, #running-req: 29, #token: 1758016, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.68, #queue-req: 3 | |
| [2026-06-26 20:16:41 DP1 TP1] Decode batch, #running-req: 29, #token: 1760448, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.76, #queue-req: 3 | |
| [2026-06-26 20:16:41 DP3 TP3] Decode batch, #running-req: 29, #token: 1759808, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.51, #queue-req: 3 | |
| [2026-06-26 20:16:42 DP2 TP2] Decode batch, #running-req: 29, #token: 1760832, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.69, #queue-req: 3 | |
| [2026-06-26 20:16:42 DP0 TP0] Decode batch, #running-req: 29, #token: 1759360, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.64, #queue-req: 3 | |
| [2026-06-26 20:16:42 DP3 TP3] Decode batch, #running-req: 29, #token: 1760832, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.62, #queue-req: 3 | |
| [2026-06-26 20:16:42 DP1 TP1] Decode batch, #running-req: 29, #token: 1761408, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.60, #queue-req: 3 | |
| [2026-06-26 20:16:43 DP2 TP2] Decode batch, #running-req: 29, #token: 1762112, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.43, #queue-req: 3 | |
| [2026-06-26 20:16:43 DP0 TP0] Decode batch, #running-req: 29, #token: 1760832, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.47, #queue-req: 3 | |
| [2026-06-26 20:16:43 DP1 TP1] Decode batch, #running-req: 29, #token: 1762816, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.53, #queue-req: 3 | |
| [2026-06-26 20:16:43 DP3 TP3] Decode batch, #running-req: 29, #token: 1761920, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.52, #queue-req: 3 | |
| [2026-06-26 20:16:44 DP2 TP2] Decode batch, #running-req: 29, #token: 1763328, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.60, #queue-req: 3 | |
| [2026-06-26 20:16:44 DP0 TP0] Decode batch, #running-req: 29, #token: 1761536, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.38, #queue-req: 3 | |
| [2026-06-26 20:16:44 DP1 TP1] Decode batch, #running-req: 29, #token: 1763840, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.24, #queue-req: 3 | |
| [2026-06-26 20:16:44 DP3 TP3] Decode batch, #running-req: 29, #token: 1763136, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.25, #queue-req: 3 | |
| [2026-06-26 20:16:45 DP2 TP2] Decode batch, #running-req: 29, #token: 1764288, token usage: 0.99, cuda graph: True, gen throughput (token/s): 992.42, #queue-req: 3 | |
| [2026-06-26 20:16:45 DP0 TP0] Decode batch, #running-req: 29, #token: 1762816, token usage: 0.99, cuda graph: True, gen throughput (token/s): 992.27, #queue-req: 3 | |
| [2026-06-26 20:16:45 DP3 TP3] Decode batch, #running-req: 29, #token: 1764288, token usage: 0.99, cuda graph: True, gen throughput (token/s): 992.54, #queue-req: 3 | |
| [2026-06-26 20:16:45 DP1 TP1] Decode batch, #running-req: 29, #token: 1764864, token usage: 0.99, cuda graph: True, gen throughput (token/s): 992.28, #queue-req: 3 | |
| [2026-06-26 20:16:46 DP2 TP2] Decode batch, #running-req: 29, #token: 1765696, token usage: 0.99, cuda graph: True, gen throughput (token/s): 993.66, #queue-req: 3 | |
| [2026-06-26 20:16:47 DP0 TP0] Decode batch, #running-req: 29, #token: 1763904, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.02, #queue-req: 3 | |
| [2026-06-26 20:16:47 DP3 TP3] Decode batch, #running-req: 29, #token: 1765440, token usage: 0.99, cuda graph: True, gen throughput (token/s): 993.80, #queue-req: 3 | |
| [2026-06-26 20:16:47 DP1 TP1] Decode batch, #running-req: 29, #token: 1766336, token usage: 0.99, cuda graph: True, gen throughput (token/s): 994.07, #queue-req: 3 | |
| [2026-06-26 20:16:48 DP2 TP2] Decode batch, #running-req: 29, #token: 1766912, token usage: 0.99, cuda graph: True, gen throughput (token/s): 993.86, #queue-req: 3 | |
| [2026-06-26 20:16:48 DP0 TP0] Decode batch, #running-req: 29, #token: 1764992, token usage: 0.99, cuda graph: True, gen throughput (token/s): 993.61, #queue-req: 3 | |
| [2026-06-26 20:16:48 DP1 TP1] Decode batch, #running-req: 29, #token: 1767168, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.63, #queue-req: 3 | |
| [2026-06-26 20:16:48 DP3 TP3] Decode batch, #running-req: 29, #token: 1766656, token usage: 0.99, cuda graph: True, gen throughput (token/s): 993.63, #queue-req: 3 | |
| [2026-06-26 20:16:49 DP2 TP2] Decode batch, #running-req: 29, #token: 1767808, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.28, #queue-req: 3 | |
| [2026-06-26 20:16:49 DP0 TP0] Decode batch, #running-req: 29, #token: 1766464, token usage: 0.99, cuda graph: True, gen throughput (token/s): 996.73, #queue-req: 3 | |
| [2026-06-26 20:16:49 DP1 TP1] Decode batch, #running-req: 29, #token: 1768576, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.91, #queue-req: 3 | |
| [2026-06-26 20:16:49 DP3 TP3] Decode batch, #running-req: 29, #token: 1767872, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.92, #queue-req: 3 | |
| [2026-06-26 20:16:50 DP2 TP2] Decode batch, #running-req: 29, #token: 1769088, token usage: 1.00, cuda graph: True, gen throughput (token/s): 997.01, #queue-req: 3 | |
| [2026-06-26 20:16:50 DP0 TP0] Decode batch, #running-req: 29, #token: 1767296, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.75, #queue-req: 3 | |
| [2026-06-26 20:16:50 DP3 TP3] Decode batch, #running-req: 29, #token: 1769088, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.63, #queue-req: 3 | |
| [2026-06-26 20:16:50 DP1 TP1] Decode batch, #running-req: 29, #token: 1769728, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.62, #queue-req: 3 | |
| [2026-06-26 20:16:51 DP2 TP2] Decode batch, #running-req: 29, #token: 1770112, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.08, #queue-req: 3 | |
| [2026-06-26 20:16:51 DP0 TP0] Decode batch, #running-req: 29, #token: 1768640, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.00, #queue-req: 3 | |
| [2026-06-26 20:16:51 DP3 TP3] Decode batch, #running-req: 29, #token: 1770112, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.21, #queue-req: 3 | |
| [2026-06-26 20:16:51 DP1 TP1] Decode batch, #running-req: 29, #token: 1770688, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.98, #queue-req: 3 | |
| [2026-06-26 20:16:52 DP2 TP2] Decode batch, #running-req: 29, #token: 1771392, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.70, #queue-req: 3 | |
| [2026-06-26 20:16:52 DP0 TP0] Decode batch, #running-req: 29, #token: 1770112, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.85, #queue-req: 3 | |
| [2026-06-26 20:16:52 DP1 TP1] Decode batch, #running-req: 29, #token: 1772096, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.82, #queue-req: 3 | |
| [2026-06-26 20:16:52 DP3 TP3] Decode batch, #running-req: 29, #token: 1771200, token usage: 1.00, cuda graph: True, gen throughput (token/s): 996.55, #queue-req: 3 | |
| [2026-06-26 20:16:53 DP2 TP2] Decode batch, #running-req: 29, #token: 1772608, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.82, #queue-req: 3 | |
| [2026-06-26 20:16:54 DP0 TP0] Decode batch, #running-req: 29, #token: 1770816, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.88, #queue-req: 3 | |
| [2026-06-26 20:16:54 DP3 TP3] Decode batch, #running-req: 29, #token: 1772416, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.99, #queue-req: 3 | |
| [2026-06-26 20:16:54 DP1 TP1] Decode batch, #running-req: 29, #token: 1773120, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.94, #queue-req: 3 | |
| [2026-06-26 20:16:55 DP2 TP2] Decode batch, #running-req: 29, #token: 1773568, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.80, #queue-req: 3 | |
| [2026-06-26 20:16:55 DP0 TP0] Decode batch, #running-req: 29, #token: 1772096, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.54, #queue-req: 3 | |
| [2026-06-26 20:16:55 DP3 TP3] Decode batch, #running-req: 29, #token: 1773568, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.48, #queue-req: 3 | |
| [2026-06-26 20:16:55 DP1 TP1] Decode batch, #running-req: 29, #token: 1774144, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.48, #queue-req: 3 | |
| [2026-06-26 20:16:56 DP2 TP2] Decode batch, #running-req: 29, #token: 1774976, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.09, #queue-req: 3 | |
| [2026-06-26 20:16:56 DP0 TP0] Decode batch, #running-req: 29, #token: 1773184, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.21, #queue-req: 3 | |
| [2026-06-26 20:16:56 DP1 TP1] Decode batch, #running-req: 29, #token: 1775616, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.16, #queue-req: 3 | |
| [2026-06-26 20:16:56 DP3 TP3] Decode batch, #running-req: 29, #token: 1774720, token usage: 1.00, cuda graph: True, gen throughput (token/s): 993.14, #queue-req: 3 | |
| [2026-06-26 20:16:56 DP1 TP1] KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_tokens_gained: 1664, #new_token_ratio: 0.0294 -> 0.7754 | |
| [2026-06-26 20:16:57 DP2 TP2] KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_tokens_gained: 1664, #new_token_ratio: 0.0294 -> 0.7866 | |
| [2026-06-26 20:16:57 DP3 TP3] KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_tokens_gained: 1664, #new_token_ratio: 0.0294 -> 0.7895 | |
| [2026-06-26 20:16:57 DP2 TP2] Decode batch, #running-req: 28, #token: 1714240, token usage: 0.97, cuda graph: True, gen throughput (token/s): 982.48, #queue-req: 4 | |
| [2026-06-26 20:16:57 DP0 TP0] Decode batch, #running-req: 29, #token: 1774272, token usage: 1.00, cuda graph: True, gen throughput (token/s): 988.41, #queue-req: 3 | |
| [2026-06-26 20:16:57 DP3 TP3] Decode batch, #running-req: 28, #token: 1713984, token usage: 0.97, cuda graph: True, gen throughput (token/s): 986.20, #queue-req: 4 | |
| [2026-06-26 20:16:57 DP1 TP1] Decode batch, #running-req: 28, #token: 1714112, token usage: 0.97, cuda graph: True, gen throughput (token/s): 962.10, #queue-req: 4 | |
| [2026-06-26 20:16:58 DP2 TP2] Decode batch, #running-req: 28, #token: 1715072, token usage: 0.97, cuda graph: True, gen throughput (token/s): 961.95, #queue-req: 4 | |
| [2026-06-26 20:16:58 DP0 TP0] Decode batch, #running-req: 29, #token: 1775744, token usage: 1.00, cuda graph: True, gen throughput (token/s): 995.46, #queue-req: 3 | |
| [2026-06-26 20:16:58 DP3 TP3] Decode batch, #running-req: 28, #token: 1715136, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.72, #queue-req: 4 | |
| [2026-06-26 20:16:58 DP1 TP1] Decode batch, #running-req: 28, #token: 1715456, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.93, #queue-req: 4 | |
| [2026-06-26 20:16:58 DP0 TP0] KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_tokens_gained: 1728, #new_token_ratio: 0.0294 -> 0.8149 | |
| [2026-06-26 20:16:59 DP2 TP2] Decode batch, #running-req: 28, #token: 1716288, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.49, #queue-req: 4 | |
| [2026-06-26 20:16:59 DP0 TP0] Decode batch, #running-req: 28, #token: 1714752, token usage: 0.97, cuda graph: True, gen throughput (token/s): 966.91, #queue-req: 4 | |
| [2026-06-26 20:16:59 DP3 TP3] Decode batch, #running-req: 28, #token: 1716352, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.45, #queue-req: 4 | |
| [2026-06-26 20:16:59 DP1 TP1] Decode batch, #running-req: 28, #token: 1716544, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.45, #queue-req: 4 | |
| [2026-06-26 20:17:00 DP2 TP2] Decode batch, #running-req: 28, #token: 1717312, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.45, #queue-req: 4 | |
| [2026-06-26 20:17:01 DP0 TP0] Decode batch, #running-req: 28, #token: 1716096, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.41, #queue-req: 4 | |
| [2026-06-26 20:17:01 DP3 TP3] Decode batch, #running-req: 28, #token: 1717312, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.43, #queue-req: 4 | |
| [2026-06-26 20:17:01 DP1 TP1] Decode batch, #running-req: 28, #token: 1717504, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.43, #queue-req: 4 | |
| [2026-06-26 20:17:02 DP2 TP2] Decode batch, #running-req: 28, #token: 1718528, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.22, #queue-req: 4 | |
| [2026-06-26 20:17:02 DP0 TP0] Decode batch, #running-req: 28, #token: 1717504, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.31, #queue-req: 4 | |
| [2026-06-26 20:17:02 DP3 TP3] Decode batch, #running-req: 28, #token: 1718336, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.20, #queue-req: 4 | |
| [2026-06-26 20:17:02 DP1 TP1] Decode batch, #running-req: 28, #token: 1718848, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.20, #queue-req: 4 | |
| [2026-06-26 20:17:03 DP2 TP2] Decode batch, #running-req: 28, #token: 1719744, token usage: 0.97, cuda graph: True, gen throughput (token/s): 956.84, #queue-req: 4 | |
| [2026-06-26 20:17:03 DP0 TP0] Decode batch, #running-req: 28, #token: 1718144, token usage: 0.97, cuda graph: True, gen throughput (token/s): 956.75, #queue-req: 4 | |
| [2026-06-26 20:17:03 DP3 TP3] Decode batch, #running-req: 28, #token: 1719552, token usage: 0.97, cuda graph: True, gen throughput (token/s): 957.05, #queue-req: 4 | |
| [2026-06-26 20:17:03 DP1 TP1] Decode batch, #running-req: 28, #token: 1719808, token usage: 0.97, cuda graph: True, gen throughput (token/s): 956.83, #queue-req: 4 | |
| [2026-06-26 20:17:04 DP2 TP2] Decode batch, #running-req: 28, #token: 1720640, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.47, #queue-req: 4 | |
| [2026-06-26 20:17:04 DP0 TP0] Decode batch, #running-req: 28, #token: 1719424, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.69, #queue-req: 4 | |
| [2026-06-26 20:17:04 DP3 TP3] Decode batch, #running-req: 28, #token: 1720640, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.49, #queue-req: 4 | |
| [2026-06-26 20:17:04 DP1 TP1] Decode batch, #running-req: 28, #token: 1720832, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.72, #queue-req: 4 | |
| [2026-06-26 20:17:05 DP2 TP2] Decode batch, #running-req: 28, #token: 1721984, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.49, #queue-req: 4 | |
| [2026-06-26 20:17:05 DP0 TP0] Decode batch, #running-req: 28, #token: 1720448, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.47, #queue-req: 4 | |
| [2026-06-26 20:17:05 DP3 TP3] Decode batch, #running-req: 28, #token: 1721792, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.50, #queue-req: 4 | |
| [2026-06-26 20:17:05 DP1 TP1] Decode batch, #running-req: 28, #token: 1722240, token usage: 0.97, cuda graph: True, gen throughput (token/s): 960.50, #queue-req: 4 | |
| [2026-06-26 20:17:06 DP2 TP2] Decode batch, #running-req: 28, #token: 1723200, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.86, #queue-req: 4 | |
| [2026-06-26 20:17:06 DP0 TP0] Decode batch, #running-req: 28, #token: 1721536, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.82, #queue-req: 4 | |
| [2026-06-26 20:17:06 DP3 TP3] Decode batch, #running-req: 28, #token: 1722944, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.79, #queue-req: 4 | |
| [2026-06-26 20:17:06 DP1 TP1] Decode batch, #running-req: 28, #token: 1723072, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.74, #queue-req: 4 | |
| [2026-06-26 20:17:07 DP2 TP2] Decode batch, #running-req: 28, #token: 1724032, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.22, #queue-req: 4 | |
| [2026-06-26 20:17:08 DP0 TP0] Decode batch, #running-req: 28, #token: 1722944, token usage: 0.97, cuda graph: True, gen throughput (token/s): 957.79, #queue-req: 4 | |
| [2026-06-26 20:17:08 DP3 TP3] Decode batch, #running-req: 28, #token: 1724096, token usage: 0.97, cuda graph: True, gen throughput (token/s): 957.57, #queue-req: 4 | |
| [2026-06-26 20:17:08 DP1 TP1] Decode batch, #running-req: 28, #token: 1724416, token usage: 0.97, cuda graph: True, gen throughput (token/s): 957.59, #queue-req: 4 | |
| [2026-06-26 20:17:09 DP2 TP2] Decode batch, #running-req: 28, #token: 1725248, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.52, #queue-req: 4 | |
| [2026-06-26 20:17:09 DP0 TP0] Decode batch, #running-req: 28, #token: 1723712, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.73, #queue-req: 4 | |
| [2026-06-26 20:17:09 DP3 TP3] Decode batch, #running-req: 28, #token: 1725312, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.24, #queue-req: 4 | |
| [2026-06-26 20:17:09 DP1 TP1] Decode batch, #running-req: 28, #token: 1725504, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.05, #queue-req: 4 | |
| [2026-06-26 20:17:10 DP2 TP2] Decode batch, #running-req: 28, #token: 1726272, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.84, #queue-req: 4 | |
| [2026-06-26 20:17:10 DP0 TP0] Decode batch, #running-req: 28, #token: 1725056, token usage: 0.97, cuda graph: True, gen throughput (token/s): 959.01, #queue-req: 4 | |
| [2026-06-26 20:17:10 DP3 TP3] Decode batch, #running-req: 28, #token: 1726272, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.55, #queue-req: 4 | |
| [2026-06-26 20:17:10 DP1 TP1] Decode batch, #running-req: 28, #token: 1726464, token usage: 0.97, cuda graph: True, gen throughput (token/s): 958.78, #queue-req: 4 | |
| [2026-06-26 20:17:10] INFO: 127.0.0.1:47994 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48446 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47940 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48498 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47774 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48594 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48116 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47992 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47846 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48614 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48090 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48394 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48204 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48174 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47742 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48554 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48140 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48298 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48210 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48052 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48670 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48020 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48370 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47900 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47694 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48638 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48324 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48250 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47806 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47784 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48652 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48340 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48514 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47914 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48118 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48058 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47998 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48376 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48094 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47852 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48410 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48462 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48596 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48150 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47956 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48022 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47812 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47682 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48266 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48686 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47754 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48626 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48176 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47860 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48570 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:47704 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:12] INFO: 127.0.0.1:48214 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:16 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.07, #running-req: 27, #queue-req: 4, #pending-token: 181226, cuda graph: False, input throughput (token/s): 899.92 | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47648 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47668 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47732 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47772 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47794 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47832 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47886 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47932 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47980 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48008 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48044 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48086 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48108 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48138 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48164 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48198 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48240 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48286 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48314 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48388 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48440 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48490 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48540 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48584 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48610 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48636 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48660 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48072 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48100 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48186 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47664 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48128 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48228 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47824 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48476 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48424 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47766 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47964 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48598 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47718 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48654 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48576 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48350 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47870 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48382 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47786 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48038 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48700 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48160 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48310 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48006 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48524 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48280 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:47918 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21] INFO: 127.0.0.1:48630 - "POST /generate HTTP/1.1" 200 OK | |
| [2026-06-26 20:17:21 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59776, #cached-token: 0, token usage: 0.10, #running-req: 28, #queue-req: 7, #pending-token: 258548, cuda graph: False, input throughput (token/s): 12152.19 | |
| [2026-06-26 20:17:26 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59904, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 16, #pending-token: 318173, cuda graph: False, input throughput (token/s): 781.90 | |
| [2026-06-26 20:17:26 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 17, #pending-token: 386692, cuda graph: False, input throughput (token/s): 776.88 | |
| [2026-06-26 20:17:26 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 59584, #cached-token: 0, token usage: 0.14, #running-req: 29, #queue-req: 15, #pending-token: 475262, cuda graph: False, input throughput (token/s): 11151.33 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| CUDA Error: CUDA_ERROR_INVALID_VALUE /workspace/include/flashinfer/trtllm/fmha/fmhaKernels.cuh 328 | |
| [2026-06-26 20:17:32 DP1 TP1] Prefill batch, #new-seq: 1, #new-token: 59456, #cached-token: 0, token usage: 0.07, #running-req: 0, #queue-req: 27, #pending-token: 731066, cuda graph: False, input throughput (token/s): 734.10 | |
| [2026-06-26 20:17:32 DP0 TP0] Prefill batch, #new-seq: 1, #new-token: 59712, #cached-token: 0, token usage: 0.10, #running-req: 1, #queue-req: 27, #pending-token: 1082151, cuda graph: False, input throughput (token/s): 11151.29 | |
| [2026-06-26 20:17:32 DP3 TP3] Prefill batch, #new-seq: 1, #new-token: 59520, #cached-token: 0, token usage: 0.10, #running-req: 1, #queue-req: 27, #pending-token: 1151587, cuda graph: False, input throughput (token/s): 11115.89 | |
| [2026-06-26 20:17:32 DP2 TP2] Prefill batch, #new-seq: 1, #new-token: 61952, #cached-token: 0, token usage: 0.17, #running-req: 30, #queue-req: 25, #pending-token: 1031931, cuda graph: False, input throughput (token/s): 11568.38 | |
| /tmp/pytorch-private/aten/src/ATen/native/cuda/TensorCompare.cu:109: _assert_async_cuda_kernel: block: [0,0,0], thread: [0,0,0] Assertion `probability tensor contains either `inf`, `nan` or element < 0` failed. | |
| Starting GPU coredump generation, set the CUDA_COREDUMP_SHOW_PROGRESS environment variable to 1 to enable more detailed output | |
| [2026-06-26 20:17:33 DP2 TP2] Scheduler hit an exception: Traceback (most recent call last): | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 4303, in run_scheduler_process | |
| scheduler.run_event_loop() | |
| ~~~~~~~~~~~~~~~~~~~~~~~~^^ | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 1501, in run_event_loop | |
| dispatch_event_loop(self) | |
| ~~~~~~~~~~~~~~~~~~~^^^^^^ | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 4168, in dispatch_event_loop | |
| scheduler.event_loop_overlap() | |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ | |
| File "/tmp/sam/uv/install/a123c83e2b667342ae15e57db7d4c0b54dfe8019a66dac295d40f1d25959ea93-3.13.13/lib/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context | |
| return func(*args, **kwargs) | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 1574, in event_loop_overlap | |
| batch = self.get_next_batch_to_run() | |
| File "/work/sam/sglang_upstream/python/sglang/srt/utils/nvtx_utils.py", line 109, in wrapper | |
| return func(*args, **kwargs) | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 2689, in get_next_batch_to_run | |
| new_batch = self.get_new_batch_prefill() | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 2749, in get_new_batch_prefill | |
| ret = self._get_new_batch_prefill_raw( | |
| prefill_delayer_single_pass=prefill_delayer_single_pass | |
| ) | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/scheduler.py", line 2973, in _get_new_batch_prefill_raw | |
| new_batch.prepare_for_extend() | |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ | |
| File "/work/sam/sglang_upstream/python/sglang/srt/managers/schedule_batch.py", line 2051, in prepare_for_extend | |
| out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend( | |
| ~~~~~~~~~~~~~~~~^ | |
| self | |
| ^^^^ | |
| ) | |
| ^ | |
| File "/work/sam/sglang_upstream/python/sglang/srt/mem_cache/common.py", line 493, in alloc_for_extend | |
| (t[-1:] if len(t) > 0 else torch.tensor([-1], device=batch.device)) | |
| ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ | |
| torch.AcceleratorError: CUDA error: unspecified launch failure | |
| Search for `cudaErrorLaunchFailure' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html for more information. | |
| CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. | |
| For debugging consider passing CUDA_LAUNCH_BLOCKING=1 | |
| Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. | |
| [2026-06-26 20:17:34] Subprocess scheduler_0 (pid=64442) crashed with exit code -3. Triggering SIGQUIT for cleanup... | |
| [2026-06-26 20:17:34] SIGQUIT received. signum=None, frame=None. It usually means one child failed. | |
| [2026-06-26 20:17:34] Sleeping 5 seconds before crash diagnostics to let GPU activity settle. | |
| [2026-06-26 20:17:39] No live scheduler processes found; skipping py-spy and CUDA coredump. | |
| [2026-06-26 20:17:39] kill_process_tree called: parent_pid=63407, include_parent=True, pid=63407 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "run_id": "upstream-glm52-dp4-fake-c256-t8-1782504221", | |
| "base_url": "http://127.0.0.1:30000", | |
| "wall_s": 837.6550970077515, | |
| "config": { | |
| "base_url": "http://127.0.0.1:30000", | |
| "run_id": "upstream-glm52-dp4-fake-c256-t8-1782504221", | |
| "num_examples": 128, | |
| "num_turns": 8, | |
| "concurrency": 256, | |
| "client_workers": 256, | |
| "client_connections": 256, | |
| "prompt_words": 12000, | |
| "followup_words": 4000, | |
| "output_len": 2048, | |
| "temperature": 1.0, | |
| "keep_chars": 6000, | |
| "stall_jitter_s": 0.0, | |
| "connect_timeout_s": 120.0, | |
| "request_timeout_s": 1800.0, | |
| "progress_every": 24, | |
| "seed": 0, | |
| "out_dir": "repro_artifacts" | |
| }, | |
| "requests": 1024, | |
| "ok_requests": 624, | |
| "client_output_tokens": 1277952, | |
| "client_prompt_tokens": 25466328, | |
| "errors": [ | |
| { | |
| "rollout_id": 22, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.82943348400295, | |
| "completed_s": 837.1899147033691, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 80, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.89829513803124, | |
| "completed_s": 837.1930646896362, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 35, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.78840356785804, | |
| "completed_s": 837.1945824623108, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 43, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.8815170568414, | |
| "completed_s": 837.1960394382477, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 29, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.0610851990059, | |
| "completed_s": 837.1975336074829, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 14, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.72280930401757, | |
| "completed_s": 837.1989991664886, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 104, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.71563231991604, | |
| "completed_s": 837.2003016471863, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 95, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.0197383519262, | |
| "completed_s": 837.201637506485, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 81, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.7096065483056, | |
| "completed_s": 837.2031235694885, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 112, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.6929130749777, | |
| "completed_s": 837.204648733139, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 37, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.06073470506817, | |
| "completed_s": 837.2084038257599, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 55, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 232.91192385926843, | |
| "completed_s": 837.2098801136017, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 23, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.138181263115257, | |
| "completed_s": 837.2112166881561, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 103, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.54900413705036, | |
| "completed_s": 837.2127568721771, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 88, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.51140665076673, | |
| "completed_s": 837.2143564224243, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 98, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.078041835222393, | |
| "completed_s": 837.2158858776093, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 21, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.081916810944676, | |
| "completed_s": 837.2174029350281, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 118, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.506144625134766, | |
| "completed_s": 837.2188055515289, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 3, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.48803795175627, | |
| "completed_s": 837.2203915119171, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 119, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.07892961986363, | |
| "completed_s": 837.2219126224518, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 109, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.070175853092223, | |
| "completed_s": 837.223450422287, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 114, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.557316205929965, | |
| "completed_s": 837.2250330448151, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 110, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.578243990894407, | |
| "completed_s": 837.226574420929, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 26, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.165874192025512, | |
| "completed_s": 837.228175163269, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 113, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.169337579980493, | |
| "completed_s": 837.2297103404999, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 54, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.57338993670419, | |
| "completed_s": 837.2313106060028, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 116, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.512873705942184, | |
| "completed_s": 837.2329008579254, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 97, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.126189841888845, | |
| "completed_s": 837.2345006465912, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 77, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.073825867846608, | |
| "completed_s": 837.236025094986, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 79, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.56572635518387, | |
| "completed_s": 837.2376079559326, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 107, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.459730751346797, | |
| "completed_s": 837.239196062088, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 93, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.11286330409348, | |
| "completed_s": 837.2434024810791, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 125, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.111918911803514, | |
| "completed_s": 837.2449443340302, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 73, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.531345599796623, | |
| "completed_s": 837.2465291023254, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 30, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.547661412041634, | |
| "completed_s": 837.2480301856995, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 66, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.067817950155586, | |
| "completed_s": 837.2502574920654, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 78, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.174838203936815, | |
| "completed_s": 837.2517952919006, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 19, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.499992900993675, | |
| "completed_s": 837.2532937526703, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 38, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.49036482302472, | |
| "completed_s": 837.2553112506866, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 48, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.100534567143768, | |
| "completed_s": 837.2566833496094, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 65, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.179540033917874, | |
| "completed_s": 837.258579492569, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 31, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.504200456198305, | |
| "completed_s": 837.2606425285339, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 69, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.61858252901584, | |
| "completed_s": 837.262179851532, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 89, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.17133001703769, | |
| "completed_s": 837.2638218402863, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 87, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.163838885258883, | |
| "completed_s": 837.2653565406799, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 86, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.51901379181072, | |
| "completed_s": 837.266845703125, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 72, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.50911992089823, | |
| "completed_s": 837.2684087753296, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 60, | |
| "turn": 4, | |
| "ok": false, | |
| "latency_s": 233.14088002499193, | |
| "completed_s": 837.269923210144, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 63, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 18.218183803837746, | |
| "completed_s": 837.2714688777924, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| }, | |
| { | |
| "rollout_id": 17, | |
| "turn": 5, | |
| "ok": false, | |
| "latency_s": 26.58583871414885, | |
| "completed_s": 837.2728791236877, | |
| "output_tokens": 0, | |
| "prompt_tokens": 0, | |
| "cached_tokens": 0, | |
| "dp_rank": null, | |
| "error": "RemoteProtocolError('Server disconnected without sending a response.')" | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment