GLM-5.2 on Baseten: AIME 2024 repro — low strict scores + slow/hung requests across serverless and dedicated endpoints (2026-07-29)

baseten_aime_repro.py

#!/usr/bin/env python3
"""Standalone repro: GLM-5.2-Fast on Baseten scores ~80% on AIME 2024 (expected: >90%).

Python 3.8+ stdlib only. Fetches the 30 AIME 2024 problems from the public
HuggingFace rows API, asks the model to answer with the final integer in
\\boxed{}, grades exact-match on the boxed integer, and writes every raw
response to aime_responses.json for debugging.

Usage:
    BASETEN_API_KEY=... python3 baseten_aime_repro.py [--model zai-org/GLM-5.2-Fast] [--concurrency 6]

Observed 2026-07-29 via our eval harness (reasoning_effort=xhigh,
max_tokens=131072, temperature default): 24/30. Failures were 4 confidently
wrong final answers + 2 responses that never produced a final answer.
"""

import argparse
import json
import os
import re
import statistics
import sys
import time
import urllib.request
from concurrent.futures import ThreadPoolExecutor

DATA_URL = (
    "https://datasets-server.huggingface.co/rows"
    "?dataset=Maxwell-Jia%2FAIME_2024&config=default&split=train&offset=0&length=100"
)
PROMPT_SUFFIX = "\n\nPlease reason step by step, and put your final answer (an integer between 0 and 999) within \\boxed{}."


def fetch_problems():
    with urllib.request.urlopen(DATA_URL, timeout=60) as r:
        rows = json.load(r)["rows"]
    return [(str(x["row"]["ID"]), x["row"]["Problem"], int(x["row"]["Answer"])) for x in rows]


def extract_boxed(text: str):
    matches = re.findall(r"\\boxed\{([^{}]+)\}", text or "")
    for m in reversed(matches):
        digits = re.sub(r"[^0-9]", "", m)
        if digits:
            return int(digits)
    return None


def one(base_url, key, model, effort, pid, problem, gold):
    body = {
        "model": model,
        "messages": [{"role": "user", "content": problem + PROMPT_SUFFIX}],
        "max_tokens": 131072,
    }
    if effort:
        body["reasoning_effort"] = effort
    req = urllib.request.Request(
        base_url.rstrip("/") + "/chat/completions",
        data=json.dumps(body).encode(),
        headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"},
    )
    t0 = time.monotonic()
    r = None
    retries_429 = 0
    for attempt in range(12):
        try:
            with urllib.request.urlopen(req, timeout=3600) as resp:
                r = json.load(resp)
            break
        except urllib.error.HTTPError as e:
            if e.code == 429:
                retries_429 += 1
                retry_after = e.headers.get("Retry-After")
                wait = float(retry_after) if retry_after else min(10 * 2 ** min(attempt, 5), 120)
                time.sleep(wait)
                continue
            return {"id": pid, "gold": gold, "error": f"HTTP {e.code}: {e.read()[:150]!r}", "seconds": round(time.monotonic() - t0, 1)}
        except Exception as e:
            return {"id": pid, "gold": gold, "error": repr(e)[:200], "seconds": round(time.monotonic() - t0, 1)}
    if r is None:
        return {"id": pid, "gold": gold, "error": f"429 after {retries_429} retries", "seconds": round(time.monotonic() - t0, 1)}
    msg = r["choices"][0]["message"]
    usage = r.get("usage", {})
    answered = extract_boxed(msg.get("content") or "")
    return {
        "id": pid,
        "gold": gold,
        "answered": answered,
        "correct": answered == gold,
        "finish_reason": r["choices"][0].get("finish_reason"),
        "completion_tokens": usage.get("completion_tokens"),
        "reasoning_tokens": (usage.get("completion_tokens_details") or {}).get("reasoning_tokens"),
        "seconds": round(time.monotonic() - t0, 1),
        "retries_429": retries_429,
        "request_id": r.get("id"),
        "raw": r,
    }


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--base-url", default="https://inference.baseten.co/v1")
    ap.add_argument("--model", default="zai-org/GLM-5.2-Fast")
    ap.add_argument("--api-key", default=os.environ.get("BASETEN_API_KEY") or os.environ.get("OPENAI_API_KEY"))
    ap.add_argument("--reasoning-effort", default="xhigh", help="pass '' to omit")
    ap.add_argument("--concurrency", type=int, default=6)
    args = ap.parse_args()
    if not args.api_key:
        print("Set BASETEN_API_KEY (or --api-key)", file=sys.stderr)
        return 2

    problems = fetch_problems()
    print(f"model={args.model} n={len(problems)} concurrency={args.concurrency} reasoning_effort={args.reasoning_effort or '(omitted)'}")
    with ThreadPoolExecutor(args.concurrency) as ex:
        results = list(
            ex.map(lambda p: one(args.base_url, args.api_key, args.model, args.reasoning_effort, *p), problems)
        )

    ok = [r for r in results if r.get("correct")]
    noans = [r for r in results if "answered" in r and r["answered"] is None]
    errs = [r for r in results if "error" in r]
    for r in sorted(results, key=lambda x: x["id"]):
        mark = "OK " if r.get("correct") else ("ERR" if "error" in r else ("-- " if r.get("answered") is None else "BAD"))
        print(
            f"  {mark} {r['id']}: answered={r.get('answered')} gold={r['gold']}"
            f" finish={r.get('finish_reason')} tokens={r.get('completion_tokens')} ({r.get('seconds')}s)"
        )
    toks = [r["completion_tokens"] for r in results if r.get("completion_tokens")]
    secs = [r["seconds"] for r in results if r.get("seconds")]
    print(f"\nSCORE: {len(ok)}/{len(results)} correct | {len(noans)} no final answer | {len(errs)} request errors")
    if toks:
        print(
            f"rollout length: mean={statistics.mean(toks):.0f} tok, median={statistics.median(toks):.0f},"
            f" max={max(toks)} | mean latency {statistics.mean(secs):.0f}s"
        )
    with open("aime_responses.json", "w") as f:
        json.dump(results, f, indent=1, ensure_ascii=False)
    print("raw responses (incl. request ids) -> aime_responses.json")
    return 0


if __name__ == "__main__":
    sys.exit(main())

baseten_aime_results.md

GLM-5.2 on Baseten — AIME 2024 results (2026-07-29)

endpoint	model	score (strict)	no boxed answer	request errors	mean latency	max latency
serverless	GLM-5.2-Fast	23/30	4	0	61s	158s
serverless	GLM-5.2	22/30	4	3 (429)	157s	315s
dedicated (womkd9kq)	GLM-5.2	21/30	7	2 never returned (1h timeout)	176s (completed only)	398s completed; 2 hung >1h

Same 30 problems, same script (baseten_aime_repro.py), reasoning_effort=xhigh, max_tokens=131072, non-streaming. Scoring: strict — final integer must be in \boxed{}.

zai-org/GLM-5.2-Fast (serverless inference.baseten.co)

• score: 23/30 | no boxed final answer: 4 | request errors: 0
• mean completion 14578 tok | mean latency 61s | max 158s
  • no-boxed-answer: problem 2024-I-4 request_id=chatcmpl-493811a54ef74a94949a465fa06b24fb (4704 tok, finish=stop)
  • no-boxed-answer: problem 2024-II-3 request_id=chatcmpl-94d764a4ca744c678c1e205cd4318814 (4979 tok, finish=stop)
  • no-boxed-answer: problem 2024-II-8 request_id=chatcmpl-e1e0189aa8f44bc0916bf58745d3f7ed (18288 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-14 request_id=chatcmpl-7b28c6728c5342a9bedb2584bbdefec9 (7209 tok, finish=stop)

zai-org/GLM-5.2 (serverless inference.baseten.co)

• score: 22/30 | no boxed final answer: 4 | request errors: 3
• mean completion 12825 tok | mean latency 130s | max 400s
  • no-boxed-answer: problem 2024-I-4 request_id=chatcmpl-6f8dca11bdfa46c7bb96e9d4eb328446 (3740 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-10 request_id=chatcmpl-0744a990ec3f41879168ea0faaed57b1 (9820 tok, finish=stop)
  • no-boxed-answer: problem 2024-II-8 request_id=chatcmpl-7c89f558c9e041d69666b25199891ac5 (16589 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-14 request_id=chatcmpl-2cb1c3b95ebd48fca399b0ec845c5f59 (12554 tok, finish=stop)
  • error: problem 2024-II-4: HTTP 502: b'{"error":"Authentication service temporarily unavailable. Please retry."}\n'
  • error: problem 2024-I-8: HTTP 504: b'\r\n<title>504 Gateway Time-out</title>\r\n\r\n<cente
  • error: problem 2024-I-11: HTTP 504: b'\r\n<title>504 Gateway Time-out</title>\r\n\r\n<cente

zai-org/GLM-5.2 (dedicated model-womkd9kq)

• score: 21/30 | no boxed final answer: 7 | request errors: 2
• mean completion 12314 tok | mean latency 176s | max 398s
  • no-boxed-answer: problem 2024-II-4 request_id=chatcmpl-d8651e26bfc849949731fde91fdbbe1a (3860 tok, finish=stop)
  • no-boxed-answer: problem 2024-II-12 request_id=chatcmpl-6e9566b084f14f28993fb5a91ca00d38 (11078 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-4 request_id=chatcmpl-3da205ef96b04f49b93b5ae5f79bee99 (3181 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-11 request_id=chatcmpl-03f71c437b4b41a39a1aa34f653c4be0 (31493 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-10 request_id=chatcmpl-e5c9cee1a9a24f0bb762a9cae67960c1 (16344 tok, finish=stop)
  • no-boxed-answer: problem 2024-II-8 request_id=chatcmpl-0f22e2769f82421da967a435437698b9 (22947 tok, finish=stop)
  • no-boxed-answer: problem 2024-I-14 request_id=chatcmpl-3d4466271c93446fad438cbee56db606 (7885 tok, finish=stop)
  • error: problem 2024-I-8: TimeoutError('The read operation timed out')
  • error: problem 2024-I-12: TimeoutError('The read operation timed out')