Nebius dedicated GLM-5.2-NVFP4 endpoint: intermittent corrupted generations — standalone repro + captured raw responses (2026-07-29)

corrupted_responses.json

[
  {
    "id": "9dfbbb6a6ebc41a19727c23b817d5213",
    "choices": [
      {
        "finish_reason": "stop",
        "index": 0,
        "logprobs": null,
        "message": {
          "content": "OK$_$_$_{}$_{\n\"$_{}$\": \"$_$_{}\"\n$_{}}$</think>OK$_{$_$_$_{}$}$\nOK$_$_{",
          "refusal": null,
          "role": "assistant",
          "annotations": null,
          "audio": null,
          "function_call": null,
          "tool_calls": null,
          "reasoning_content": "1"
        },
        "matched_stop": 154827
      }
    ],
    "created": 1785335329,
    "model": "dedicated/thinkingmachines/GLM-5.2-NVFP4-0i0tLg3U10Fl",
    "object": "chat.completion",
    "moderation": null,
    "service_tier": null,
    "system_fingerprint": null,
    "usage": {
      "completion_tokens": 42,
      "prompt_tokens": 18,
      "total_tokens": 60,
      "completion_tokens_details": {
        "accepted_prediction_tokens": null,
        "audio_tokens": null,
        "reasoning_tokens": 5,
        "rejected_prediction_tokens": null
      },
      "prompt_tokens_details": null,
      "prompt_cache_hit_tokens": 0,
      "prompt_cache_miss_tokens": 18,
      "reasoning_tokens": 5
    },
    "metadata": {
      "weight_version": "default"
    }
  },
  {
    "id": "371c73354b7948f08efc671cd5ca8361",
    "choices": [
      {
        "finish_reason": "stop",
        "index": 0,
        "logprobs": null,
        "message": {
          "content": "OK$_{}$$_$_$_$_{}$</think>OK$_{}$$^{}$</think>OK$_{}$</think>OK$_{}$//OK",
          "refusal": null,
          "role": "assistant",
          "annotations": null,
          "audio": null,
          "function_call": null,
          "tool_calls": null,
          "reasoning_content": "1"
        },
        "matched_stop": 154827
      }
    ],
    "created": 1785335331,
    "model": "dedicated/thinkingmachines/GLM-5.2-NVFP4-0i0tLg3U10Fl",
    "object": "chat.completion",
    "moderation": null,
    "service_tier": null,
    "system_fingerprint": null,
    "usage": {
      "completion_tokens": 41,
      "prompt_tokens": 18,
      "total_tokens": 59,
      "completion_tokens_details": {
        "accepted_prediction_tokens": null,
        "audio_tokens": null,
        "reasoning_tokens": 10,
        "rejected_prediction_tokens": null
      },
      "prompt_tokens_details": null,
      "prompt_cache_hit_tokens": 0,
      "prompt_cache_miss_tokens": 18,
      "reasoning_tokens": 10
    },
    "metadata": {
      "weight_version": "default"
    }
  },
  {
    "id": "291e763edfd54b9391c59d396023a6e4",
    "choices": [
      {
        "finish_reason": "stop",
        "index": 0,
        "logprobs": null,
        "message": {
          "content": "OK$_{}$_Ok.</think>OK$_{}$_Okay._OK_</think>OK$_{}$_OK!_OK.</think>OK",
          "refusal": null,
          "role": "assistant",
          "annotations": null,
          "audio": null,
          "function_call": null,
          "tool_calls": null,
          "reasoning_content": "1"
        },
        "matched_stop": 154827
      }
    ],
    "created": 1785335331,
    "model": "dedicated/thinkingmachines/GLM-5.2-NVFP4-0i0tLg3U10Fl",
    "object": "chat.completion",
    "moderation": null,
    "service_tier": null,
    "system_fingerprint": null,
    "usage": {
      "completion_tokens": 34,
      "prompt_tokens": 18,
      "total_tokens": 52,
      "completion_tokens_details": {
        "accepted_prediction_tokens": null,
        "audio_tokens": null,
        "reasoning_tokens": 6,
        "rejected_prediction_tokens": null
      },
      "prompt_tokens_details": null,
      "prompt_cache_hit_tokens": 0,
      "prompt_cache_miss_tokens": 18,
      "reasoning_tokens": 6
    },
    "metadata": {
      "weight_version": "default"
    }
  }
]

nebius_corruption_repro.py

#!/usr/bin/env python3
"""Standalone repro: intermittent corrupted generations on the Nebius dedicated
GLM-5.2-NVFP4 endpoint.

Requires only Python 3.8+ stdlib. Sends N identical, trivial, non-streaming
chat completions ("Say OK and nothing else.", no tools, no system prompt) at a
given concurrency and classifies each response. A healthy endpoint answers
"OK" in well under 20 tokens; corrupted responses run to the max_tokens cap or
emit structurally impossible content.

Usage:
    NEBIUS_API_KEY=... python3 probe_corruption.py [N] [CONCURRENCY]
    python3 probe_corruption.py --api-key ... --n 48 --concurrency 8

Observed on 2026-07-28/29 against the dedicated endpoint (defaults below):
  * ~10-19% of responses corrupted at concurrency 8 (24/192 + 9/48 across bursts)
  * ~2-3% corrupted even fully sequential
  * explicit temperature/top_p make no difference
  * rate rises under real workload; agentic SWE-bench runs at ~130 concurrent
    requests saw 60-70% of trajectories die on unparseable output, plus
    requests that hang indefinitely
  * an identically-driven GLM-5.2 endpoint on another provider: 0/48 corrupted

Real captured examples (content field, verbatim):
    'OK</think>九月开学季，秋季运动会在清华大学有哪些 '        <- </think> leak + unrelated Chinese
    '$_OK</arg_value>Eval Mode</think>OK$_SYSTEM$</think>...'  <- tool-call XML with NO tools defined
    'OK$_{}$_$_$_</think>OK$_{}$'                              <- junk interleave
    '## DIS$_{C$_{USSION}\\n\\nI'll list the files...'         <- junk spliced mid-word
    reasoning: '1 my\\n\\nThe user{_ is{_ asking me to say'    <- token interleave in reasoning

Every corrupted response's full raw JSON (including the server-assigned request
`id`, for provider-side log correlation) is written to corrupted_responses.json.
"""

import argparse
import json
import os
import sys
import urllib.request
from collections import Counter
from concurrent.futures import ThreadPoolExecutor

DEFAULT_BASE_URL = "https://api.studio.nebius.com/v1"
DEFAULT_MODEL = "dedicated/thinkingmachines/GLM-5.2-NVFP4-0i0tLg3U10Fl"
PROMPT = "Say OK and nothing else."
MAX_TOKENS = 512  # healthy responses use <20 tokens; corrupted ones hit this cap
REQUEST_TIMEOUT = 300  # corrupted generations decode ~21 tok/s to the cap; hangs beyond this are counted


def classify(response: dict) -> str:
    choice = response["choices"][0]
    msg = choice["message"]
    content = msg.get("content") or ""
    reasoning = msg.get("reasoning_content") or ""
    corrupted = (
        # this prompt can never legitimately reach the cap
        choice["finish_reason"] == "length"
        # reasoning-wrapper tags can never legitimately appear inside content
        or "</think>" in content
        # tool-call XML can never appear: the request defines no tools
        or "</arg_value>" in content
        # degenerate backtick runs seen in corrupted reasoning streams
        or "``````" in reasoning
        or "``````" in content
    )
    return "CORRUPTED" if corrupted else "OK"


def one(base_url: str, key: str, model: str, i: int):
    body = {
        "model": model,
        "messages": [{"role": "user", "content": PROMPT}],
        "max_tokens": MAX_TOKENS,
    }
    req = urllib.request.Request(
        base_url.rstrip("/") + "/chat/completions",
        data=json.dumps(body).encode(),
        headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"},
    )
    try:
        with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
            r = json.load(resp)
    except Exception as e:
        return ("REQUEST_FAILED_OR_HUNG", {"error": repr(e)})
    return (classify(r), r)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("n_pos", nargs="?", type=int, help="number of requests (positional)")
    ap.add_argument("conc_pos", nargs="?", type=int, help="concurrency (positional)")
    ap.add_argument("--n", type=int, default=48)
    ap.add_argument("--concurrency", type=int, default=8)
    ap.add_argument("--base-url", default=os.environ.get("NEBIUS_BASE_URL", DEFAULT_BASE_URL))
    ap.add_argument("--model", default=os.environ.get("NEBIUS_MODEL", DEFAULT_MODEL))
    ap.add_argument("--api-key", default=os.environ.get("NEBIUS_API_KEY"))
    args = ap.parse_args()
    n = args.n_pos or args.n
    conc = args.conc_pos or args.concurrency
    if not args.api_key:
        print("Set NEBIUS_API_KEY or pass --api-key", file=sys.stderr)
        return 2

    print(f"endpoint={args.base_url}  model={args.model}")
    print(f"n={n} concurrency={conc} prompt={PROMPT!r} max_tokens={MAX_TOKENS}")
    with ThreadPoolExecutor(conc) as ex:
        results = list(ex.map(lambda i: one(args.base_url, args.api_key, args.model, i), range(n)))

    counts = Counter(tag for tag, _ in results)
    bad = [r for tag, r in results if tag != "OK"]
    print(dict(counts))
    for tag, r in results:
        if tag == "CORRUPTED":
            m = r["choices"][0]["message"]
            print(
                f"  id={r.get('id')} finish={r['choices'][0]['finish_reason']}"
                f" content={(m.get('content') or '')[:60]!r}"
                f" reasoning[:60]={(m.get('reasoning_content') or '')[:60]!r}"
            )
        elif tag == "REQUEST_FAILED_OR_HUNG":
            print("  REQUEST_FAILED_OR_HUNG", r["error"])

    n_bad = sum(v for k, v in counts.items() if k != "OK")
    print(f"\n{n_bad}/{n} requests corrupted/failed ({100 * n_bad / n:.1f}%)")
    if bad:
        with open("corrupted_responses.json", "w") as f:
            json.dump(bad, f, indent=2, ensure_ascii=False)
        print("full raw JSON (incl. server request ids) -> corrupted_responses.json")
    else:
        print("no corruption in this burst; rerun or raise N/CONCURRENCY")
    return 0


if __name__ == "__main__":
    sys.exit(main())