namgyu-youn · June 13, 2026 07:07 · namgyu-youn · Jun 13, 2026
diff --git a/run_benchmark.py b/run_benchmark.py
 #!/usr/bin/env python3
 """
 Benchmark: video token estimation for Qwen3-VL using Video-MME.
 Usage: python run_benchmark.py --base-url http://localhost:8000 [--num-videos 10]
 Dependencies: pip install requests huggingface_hub datasets yt-dlp transformers
 Reference: github.com/QwenLM/Qwen3-VL (vision_process.py)
 """

 import argparse, base64, math, time
 import numpy as np, requests, yt_dlp
 from datasets import load_dataset
 from transformers.models.qwen3_vl.video_processing_qwen3_vl import smart_resize as _smart_resize

 MODEL           = "Qwen/Qwen3-VL-30B-A3B-Instruct"
 PATCH_SIZE      = 16            # Qwen3-VL ViT patch size
 MERGE_SIZE      = 2             # spatial merge
 TEMPORAL_PATCH  = 2             # temporal grouping
 VP_MIN_PIXELS   = 4_096         # video_processor.size["shortest_edge"]
 VP_MAX_PIXELS   = 25_165_824    # video_processor.size["longest_edge"]
 VLLM_NUM_FRAMES = 32            # VideoMediaIO default num_frames cap
 SAMPLE_FPS      = 2.0
 FPS_MIN_FRAMES  = 4
 FPS_MAX_FRAMES  = 768
 PROMPT_OVERHEAD = 13            # chat template + "Describe this video briefly."
 ITERS           = 3


 def estimate(total_frames, src_fps, w, h):
    # vLLM VideoBackend caps loaded frames at VLLM_NUM_FRAMES=32.
    # ≤32 total → all frames loaded, do_sample_frames=True  → HF processor fps-samples
    # >32 total → 32 frames pre-sampled, do_sample_frames=False → HF uses all 32 as-is
    if total_frames <= VLLM_NUM_FRAMES:
        nframes = int(total_frames / src_fps * SAMPLE_FPS)
        nframes = max(FPS_MIN_FRAMES, min(FPS_MAX_FRAMES, nframes))
    else:
        nframes = VLLM_NUM_FRAMES

    h_bar, w_bar = _smart_resize(nframes, h, w, min_pixels=VP_MIN_PIXELS, max_pixels=VP_MAX_PIXELS)
    t_bar  = math.ceil(nframes / TEMPORAL_PATCH) * TEMPORAL_PATCH
    grid_t = t_bar  // TEMPORAL_PATCH
    grid_h = h_bar  // PATCH_SIZE
    grid_w = w_bar  // PATCH_SIZE
    visual = grid_t * grid_h * grid_w // (MERGE_SIZE ** 2)

    # Per-grid overhead: "<X.X seconds>" = 6/7/8 tokens for X <10/<100/≥100,
    # plus vision_start (1) + vision_end (1).
    # For >32-frame videos use exact linspace frame positions (matching vLLM's pre-sampling).
    if total_frames > VLLM_NUM_FRAMES:
        frame_idx  = np.linspace(0, total_frames - 1, VLLM_NUM_FRAMES, dtype=int).tolist()
        raw_ts     = [idx / src_fps for idx in frame_idx]
        timestamps = [(raw_ts[i] + raw_ts[i + 1]) / 2
                      for i in range(0, VLLM_NUM_FRAMES, TEMPORAL_PATCH)]
    else:
        duration   = total_frames / src_fps
        timestamps = [(i + 0.5) * duration / grid_t for i in range(grid_t)]
    ts_tokens = sum(6 + (t >= 10) + (t >= 100) for t in timestamps)
    return visual + ts_tokens + grid_t * 2 + PROMPT_OVERHEAD


 def query(base_url, path):
    if path.startswith("http"):
        data = requests.get(path, timeout=120).content
    else:
        with open(path, "rb") as f:
            data = f.read()
    b64 = base64.b64encode(data).decode()
    r = requests.post(f"{base_url}/v1/chat/completions", timeout=180, json={
        "model": MODEL,
        "messages": [{"role": "user", "content": [
            {"type": "video_url", "video_url": {"url": f"data:video/mp4;base64,{b64}"}},
            {"type": "text", "text": "Describe this video briefly."},
        ]}],
        "max_tokens": 1,   # minimise generation; we only need prompt_tokens from usage
    })
    r.raise_for_status()
    return r.json()["usage"]["prompt_tokens"]


 def wait_for_server(base_url, timeout=300):
    for _ in range(timeout // 5):
        try:
            if requests.get(f"{base_url}/health", timeout=5).status_code == 200:
                print("Server ready.")
                return
        except Exception:
            pass
        time.sleep(5)
    raise RuntimeError("Server not ready.")


 def resolve_stream_url(youtube_url):
    """Return (direct_cdn_url, video_id, total_frames, fps, width, height)."""
    opts = {"format": "best[ext=mp4]/best", "quiet": True, "no_warnings": True}
    with yt_dlp.YoutubeDL(opts) as ydl:
        info = ydl.extract_info(youtube_url, download=False)
    fps = info.get("fps") or 30.0
    w, h = info["width"], info["height"]
    n = int(info["duration"] * fps)
    return info["url"], info["id"], n, fps, w, h


 def main():
    p = argparse.ArgumentParser()
    p.add_argument("--base-url",   default="http://localhost:8000")
    p.add_argument("--num-videos", type=int, default=10)
    args = p.parse_args()
    base_url = args.base_url.rstrip("/")

    wait_for_server(base_url)
    ds = load_dataset("lmms-lab/Video-MME", split="test")
    # deduplicate: one entry per unique videoID
    seen, unique_rows = set(), []
    for row in ds:
        if row["videoID"] not in seen:
            seen.add(row["videoID"])
            unique_rows.append(row)
        if len(unique_rows) == args.num_videos:
            break

    rows = []
    for row in unique_rows:
        vid = row["videoID"]
        try:
            stream_url, _, n, fps, w, h = resolve_stream_url(row["url"])
        except Exception as e:
            print(f"SKIP {vid}: {e}")
            continue

        est = estimate(n, fps, w, h)
        print(f"\n=== {vid} | {n/fps:.1f}s {fps:.1f}fps {w}x{h} est={est} ===")
        actuals = []
        for i in range(ITERS):
            print(f"  [iter {i+1}]", end=" ", flush=True)
            try:
                actual = query(base_url, stream_url)
                print(f"api={actual} diff={abs(est-actual)}")
                actuals.append(actual)
            except Exception as e:
                print(f"ERROR: {e}")
            time.sleep(0.3)
        if actuals:
            avg = sum(actuals) / len(actuals)
            rows.append((vid, n/fps, fps, w, h, est, avg, abs(est-avg)))

    if not rows:
        return
    print(f"\n{'Video':<30} {'Dur':>6} {'FPS':>5} {'Res':>10} {'Est':>8} {'Actual':>8} {'Diff':>8} {'Diff%':>7}")
    print("-" * 88)
    for name, dur, fps, w, h, est, actual, diff in rows:
        pct = diff / actual * 100 if actual else float("nan")
        print(f"{name[:28]:<30} {dur:>6.1f} {fps:>5.1f} {w}x{h:<5} {est:>8.0f} {actual:>8.0f} {diff:>8.2f} {pct:>6.1f}%")


 if __name__ == "__main__":
    main()
	#!/usr/bin/env python3
	"""
	Benchmark: video token estimation for Qwen3-VL using Video-MME.
	Usage: python run_benchmark.py --base-url http://localhost:8000 [--num-videos 10]
	Dependencies: pip install requests huggingface_hub datasets yt-dlp transformers
	Reference: github.com/QwenLM/Qwen3-VL (vision_process.py)
	"""

	import argparse, base64, math, time
	import numpy as np, requests, yt_dlp
	from datasets import load_dataset
	from transformers.models.qwen3_vl.video_processing_qwen3_vl import smart_resize as _smart_resize

	MODEL = "Qwen/Qwen3-VL-30B-A3B-Instruct"
	PATCH_SIZE = 16 # Qwen3-VL ViT patch size
	MERGE_SIZE = 2 # spatial merge
	TEMPORAL_PATCH = 2 # temporal grouping
	VP_MIN_PIXELS = 4_096 # video_processor.size["shortest_edge"]
	VP_MAX_PIXELS = 25_165_824 # video_processor.size["longest_edge"]
	VLLM_NUM_FRAMES = 32 # VideoMediaIO default num_frames cap
	SAMPLE_FPS = 2.0
	FPS_MIN_FRAMES = 4
	FPS_MAX_FRAMES = 768
	PROMPT_OVERHEAD = 13 # chat template + "Describe this video briefly."
	ITERS = 3


	def estimate(total_frames, src_fps, w, h):
	# vLLM VideoBackend caps loaded frames at VLLM_NUM_FRAMES=32.
	# ≤32 total → all frames loaded, do_sample_frames=True → HF processor fps-samples
	# >32 total → 32 frames pre-sampled, do_sample_frames=False → HF uses all 32 as-is
	if total_frames <= VLLM_NUM_FRAMES:
	nframes = int(total_frames / src_fps * SAMPLE_FPS)
	nframes = max(FPS_MIN_FRAMES, min(FPS_MAX_FRAMES, nframes))
	else:
	nframes = VLLM_NUM_FRAMES

	h_bar, w_bar = _smart_resize(nframes, h, w, min_pixels=VP_MIN_PIXELS, max_pixels=VP_MAX_PIXELS)
	t_bar = math.ceil(nframes / TEMPORAL_PATCH) * TEMPORAL_PATCH
	grid_t = t_bar // TEMPORAL_PATCH
	grid_h = h_bar // PATCH_SIZE
	grid_w = w_bar // PATCH_SIZE
	visual = grid_t * grid_h * grid_w // (MERGE_SIZE ** 2)

	# Per-grid overhead: "<X.X seconds>" = 6/7/8 tokens for X <10/<100/≥100,
	# plus vision_start (1) + vision_end (1).
	# For >32-frame videos use exact linspace frame positions (matching vLLM's pre-sampling).
	if total_frames > VLLM_NUM_FRAMES:
	frame_idx = np.linspace(0, total_frames - 1, VLLM_NUM_FRAMES, dtype=int).tolist()
	raw_ts = [idx / src_fps for idx in frame_idx]
	timestamps = [(raw_ts[i] + raw_ts[i + 1]) / 2
	for i in range(0, VLLM_NUM_FRAMES, TEMPORAL_PATCH)]
	else:
	duration = total_frames / src_fps
	timestamps = [(i + 0.5) * duration / grid_t for i in range(grid_t)]
	ts_tokens = sum(6 + (t >= 10) + (t >= 100) for t in timestamps)
	return visual + ts_tokens + grid_t * 2 + PROMPT_OVERHEAD


	def query(base_url, path):
	if path.startswith("http"):
	data = requests.get(path, timeout=120).content
	else:
	with open(path, "rb") as f:
	data = f.read()
	b64 = base64.b64encode(data).decode()
	r = requests.post(f"{base_url}/v1/chat/completions", timeout=180, json={
	"model": MODEL,
	"messages": [{"role": "user", "content": [
	{"type": "video_url", "video_url": {"url": f"data:video/mp4;base64,{b64}"}},
	{"type": "text", "text": "Describe this video briefly."},
	]}],
	"max_tokens": 1, # minimise generation; we only need prompt_tokens from usage
	})
	r.raise_for_status()
	return r.json()["usage"]["prompt_tokens"]


	def wait_for_server(base_url, timeout=300):
	for _ in range(timeout // 5):
	try:
	if requests.get(f"{base_url}/health", timeout=5).status_code == 200:
	print("Server ready.")
	return
	except Exception:
	pass
	time.sleep(5)
	raise RuntimeError("Server not ready.")


	def resolve_stream_url(youtube_url):
	"""Return (direct_cdn_url, video_id, total_frames, fps, width, height)."""
	opts = {"format": "best[ext=mp4]/best", "quiet": True, "no_warnings": True}
	with yt_dlp.YoutubeDL(opts) as ydl:
	info = ydl.extract_info(youtube_url, download=False)
	fps = info.get("fps") or 30.0
	w, h = info["width"], info["height"]
	n = int(info["duration"] * fps)
	return info["url"], info["id"], n, fps, w, h


	def main():
	p = argparse.ArgumentParser()
	p.add_argument("--base-url", default="http://localhost:8000")
	p.add_argument("--num-videos", type=int, default=10)
	args = p.parse_args()
	base_url = args.base_url.rstrip("/")

	wait_for_server(base_url)
	ds = load_dataset("lmms-lab/Video-MME", split="test")
	# deduplicate: one entry per unique videoID
	seen, unique_rows = set(), []
	for row in ds:
	if row["videoID"] not in seen:
	seen.add(row["videoID"])
	unique_rows.append(row)
	if len(unique_rows) == args.num_videos:
	break

	rows = []
	for row in unique_rows:
	vid = row["videoID"]
	try:
	stream_url, _, n, fps, w, h = resolve_stream_url(row["url"])
	except Exception as e:
	print(f"SKIP {vid}: {e}")
	continue

	est = estimate(n, fps, w, h)
	print(f"\n=== {vid} \| {n/fps:.1f}s {fps:.1f}fps {w}x{h} est={est} ===")
	actuals = []
	for i in range(ITERS):
	print(f" [iter {i+1}]", end=" ", flush=True)
	try:
	actual = query(base_url, stream_url)
	print(f"api={actual} diff={abs(est-actual)}")
	actuals.append(actual)
	except Exception as e:
	print(f"ERROR: {e}")
	time.sleep(0.3)
	if actuals:
	avg = sum(actuals) / len(actuals)
	rows.append((vid, n/fps, fps, w, h, est, avg, abs(est-avg)))

	if not rows:
	return
	print(f"\n{'Video':<30} {'Dur':>6} {'FPS':>5} {'Res':>10} {'Est':>8} {'Actual':>8} {'Diff':>8} {'Diff%':>7}")
	print("-" * 88)
	for name, dur, fps, w, h, est, actual, diff in rows:
	pct = diff / actual * 100 if actual else float("nan")
	print(f"{name[:28]:<30} {dur:>6.1f} {fps:>5.1f} {w}x{h:<5} {est:>8.0f} {actual:>8.0f} {diff:>8.2f} {pct:>6.1f}%")


	if __name__ == "__main__":
	main()
No results found