C3 fresh held-out paired evaluation protocol v19

protocol-v19-fresh-confirmatory-preregistration.json

{
  "protocol": "held-out-paired-c3-v19",
  "registered_at_utc": "2026-07-16T09:15:59Z",
  "status": "prepared before any v19 arm or judge inference",
  "objective": {
    "fresh_case_family_count": 3,
    "valid_pair_count": 3,
    "valid_arm_count": 6,
    "agent_model": "gpt-5.6-sol",
    "agent_effort": "low",
    "required_surface_coverage_lift_target": 0.2,
    "blocking_false_claim_lift_max": 0
  },
  "held_out_design": {
    "families": [
      "retry-safe multi-step reconciliation",
      "historical period semantics",
      "failure-safe relationship reassignment"
    ],
    "private_cases_sha256": "159a9cbeb04250317368075c3d870cb5ee6e9e5688f0b2014e22b17e3b0afb69",
    "private_protocol_sha256": "e49e540b877306529845f57390870b8ef86ae9dba299bfbb2a3d65c22b59d8f0",
    "observed_v17_case_reuse_count": 0,
    "project_specific_public_field_count": 0
  },
  "controls": {
    "standardized_baseline_instruction_sha256": "c67682acb11535517ba2c70c08eb2de907c506dabc3e2f2a5c69e0f3fafd7ac9",
    "treatment_instruction_sha256": "86dafd2a6d2c1766276aa5b05ea285cce7218bf9719564be8b6a6a1c2d99ec8b",
    "frozen_c3_binary_sha256": "73aa6df4572327bedf1ff87c0fa2558c0111ca7558856358b6eac43fc0f443d0",
    "frozen_c3_sweep_sha256": "ab68bf44f04d9a2609f0eb79b73c9e2c802510c99c25579d42450c44934833ec",
    "condition_order_seed": 4101,
    "max_tool_calls_per_arm": 10,
    "max_total_tokens_per_arm": 500000,
    "max_cost_usd_per_arm": 0.3,
    "timeout_seconds_per_arm": 600,
    "arm_reserve_usd": 1.8
  },
  "judging": {
    "judge_count": 2,
    "distinct_provider_count": 2,
    "condition_blind": true,
    "packet_scope": "claim-only; no source repository or tools",
    "judge_1": "claude-fable-5 low, hard cap 0.80 USD",
    "judge_2": "gpt-5.6-sol high, reserve 0.50 USD",
    "conditional_adjudicator": "gpt-5.6-luna high, reserve 0.70 USD",
    "claim_labels": ["supported", "missing", "contradicted"],
    "primary_score": "supported gold claims divided by gold claim count",
    "blocking_false_claim_count": "contradicted gold claims plus supported forbidden claims",
    "preference_rule": "fewer blocking false claims wins; if equal, higher required-surface coverage wins; otherwise tie",
    "raw_preference_agreement_target": 0.6666666667,
    "adjudication": "Every preference disagreement or blocking-false-count disagreement is adjudicated from a blind packet containing anonymous judgments.",
    "reserve_usd": 2.0
  },
  "analysis": {
    "direction": "with skill minus without skill",
    "primary_metric": "paired required-surface coverage lift",
    "safety_metric": "paired blocking false claim lift",
    "secondary_metrics": ["elapsed time", "tokens", "cost", "tool calls"],
    "bootstrap_seed": 20260719,
    "bootstrap_resamples": 100000,
    "confidence_level": 0.95,
    "small_sample_rule": "Report the three-pair estimate and interval. If the interval crosses zero, do not claim population-level certainty even if the point target is met."
  },
  "anti_goals": {
    "new_confirmatory_cost_usd_max": 4.0,
    "worst_case_reserved_usd": 3.8,
    "study_protocol_deviation_count_max": 0,
    "public_project_specific_field_count_max": 0,
    "observed_answer_reuse_as_confirmatory_count_max": 0,
    "anti_goal_bypass_or_dishonesty_count_max": 0
  },
  "stopping_rule": "Stop on any invalid arm, missing registered judge output, budget admission failure, privacy breach, or protocol deviation. Do not substitute or rerun a failed registered judge. The OKRA objective passes only when lift >= 0.20, false-claim lift <= 0, two judges exist, raw agreement >= 2/3, every required disagreement is adjudicated, and all walls hold."
}