C3 held-out paired evaluation v16 preregistration

protocol-v16-held-out-preregistration.json

{
  "protocol": "held-out-paired-c3-v16",
  "registered_at_utc": "2026-07-16T07:55:48Z",
  "status": "registered_before any v16 arm or judge inference",
  "supersedes": {
    "protocols": ["held-out-paired-c3-v12", "held-out-paired-c3-v13", "held-out-paired-c3-v14", "held-out-paired-c3-v15"],
    "reason": "V12 and v13 stopped before inference on schema checks. V14 produced no retained answer after a tool-call kill. V15 produced no retained answer after crossing its token wall. V16 changes only operational token and cost ceilings; held-out cases, private rubric commitments, conditions, judge rule, analysis, and seeds remain fixed.",
    "prior_successor_study_cost_usd": {
      "v14_conservative_reserve": 0.4,
      "v15_measured": 0.31912,
      "total": 0.71912
    }
  },
  "objective": {
    "conceptual_case_family_count": 3,
    "randomized_repeats_per_condition": 3,
    "valid_pair_count": 9,
    "valid_arm_count": 18,
    "agent_model": "gpt-5.6-sol",
    "agent_effort": "low",
    "raw_pair_preference_agreement_target": 0.8
  },
  "held_out_design": {
    "families": [
      "authorization policy change impact",
      "attachment lifecycle change impact",
      "shared UI contract change impact"
    ],
    "private_case_commitments_sha256": [
      "fa9f19dab5d58b36bcd6590bbd285a23ccbf06797e4d6a9c559425833505b6fa",
      "278a88ae3696a860aac22d87e0d56606ee1b4579d468ee36dd5ee217cb9b6c7a",
      "c604a7f6c4636157f325fce9a3295a1bddae8112975ec3028caac2affe6f5dc8"
    ],
    "private_protocol_commitment_sha256": "f8c943fa651c33b691f0998b2dc766b318655b56f742ffd65b0da68cb81669de",
    "project_specific_public_field_count": 0
  },
  "controls": {
    "standardized_baseline_generation_count": 1,
    "standardized_baseline_instruction_sha256": "c67682acb11535517ba2c70c08eb2de907c506dabc3e2f2a5c69e0f3fafd7ac9",
    "treatment_instruction_sha256": "86dafd2a6d2c1766276aa5b05ea285cce7218bf9719564be8b6a6a1c2d99ec8b",
    "frozen_runtime_sha256": "e346474c62fa093daac590734c6adcb28e1dbf7164500952003db48502a6901b",
    "condition_order_seed_commitment": "seeds 2101 through 2109",
    "max_tool_calls_per_arm": 10,
    "max_total_tokens_per_arm": 500000,
    "max_cost_usd_per_arm": 0.35,
    "timeout_seconds_per_arm": 600
  },
  "judging": {
    "judge_count": 2,
    "distinct_provider_count": 2,
    "condition_blind": true,
    "opposite_alias_randomization": true,
    "claim_labels": ["supported", "missing", "contradicted"],
    "score_formula": "2 * supported gold claims - 3 * contradicted gold claims - 4 * supported forbidden claims",
    "pick_rule": "higher score, then fewer supported forbidden claims, then higher change-usefulness score, otherwise tie",
    "agreement_metric": "same condition-level pair preference after private alias remapping",
    "agreement_target": 0.8,
    "adjudication": "Every preference, claim-label, or material usefulness disagreement receives condition-blind rule-consistent adjudication.",
    "reserved_cost_usd": 2.98
  },
  "analysis": {
    "direction": "with skill minus without skill",
    "bootstrap_seed": 20260717,
    "bootstrap_resamples": 100000,
    "confidence_level": 0.95,
    "metrics": ["quality", "correctness", "elapsed time", "tokens", "cost"]
  },
  "anti_goals": {
    "successor_study_cumulative_cost_usd_max": 10.0,
    "v16_valid_arm_protocol_deviation_count_max": 0,
    "v16_study_protocol_deviation_count_max": 0,
    "public_project_specific_field_count_max": 0,
    "anti_goal_bypass_or_dishonesty_count_max": 0,
    "worst_case_admitted_total_usd": 9.99912
  },
  "stopping_rule": "No paper-grade claim unless every v16 objective item and anti-goal passes from replayable evidence. Failed or invalid v16 inference attempts remain in the cumulative successor-study ledger."
}