Created
September 29, 2026 23:51
-
-
Save seyi/d08938a74dbf14a63be802b1515fc759 to your computer and use it in GitHub Desktop.
DeepEval judge provenance pilot aggregate analysis
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "M1_coverage": { | |
| "by_call_type": { | |
| "geval_score": { | |
| "captured": 60, | |
| "wire": 60 | |
| }, | |
| "geval_steps": { | |
| "captured": 60, | |
| "wire": 60 | |
| }, | |
| "tc_extract": { | |
| "captured": 60, | |
| "wire": 60 | |
| }, | |
| "tc_verdict": { | |
| "captured": 60, | |
| "wire": 60 | |
| } | |
| }, | |
| "metricdata_forwarding": { | |
| "agree": 120, | |
| "exchanges_agree": 120 | |
| }, | |
| "units": { | |
| "complete": 120 | |
| }, | |
| "units_with_unmatched_pairs": [] | |
| }, | |
| "M2_prompt_fidelity": { | |
| "captures": 240, | |
| "captures_matching_no_sent_prompt": 0, | |
| "pairs": 240, | |
| "prompt_identical": 240, | |
| "response_format_in_capture": 0, | |
| "sampling_params_in_capture": 0, | |
| "system_message_sent": 0 | |
| }, | |
| "M3_response_fidelity": { | |
| "geval_score": { | |
| "exact_response_body:applicable": 60, | |
| "exact_response_body:captured": 0, | |
| "exact_response_body:captured_in_exchange": 60, | |
| "exchange_present": 60, | |
| "exchange_status_live": 60, | |
| "finish_reason:applicable": 60, | |
| "finish_reason:captured": 0, | |
| "finish_reason:captured_in_exchange": 60, | |
| "identical_to_wire_content": 60, | |
| "logprobs:applicable": 60, | |
| "logprobs:captured": 0, | |
| "logprobs:captured_in_exchange": 60, | |
| "model:applicable": 60, | |
| "model:captured": 0, | |
| "model:captured_in_exchange": 60, | |
| "pairs": 60, | |
| "provider_request_id:applicable": 60, | |
| "provider_request_id:captured": 0, | |
| "provider_request_id:captured_in_exchange": 60, | |
| "response_body_sha256:applicable": 60, | |
| "response_body_sha256:captured": 0, | |
| "response_body_sha256:captured_in_exchange": 60, | |
| "response_headers:applicable": 60, | |
| "response_headers:captured": 0, | |
| "response_headers:captured_in_exchange": 0, | |
| "round_trips": 60, | |
| "system_fingerprint:applicable": 60, | |
| "system_fingerprint:captured": 0, | |
| "system_fingerprint:captured_in_exchange": 60, | |
| "usage:applicable": 60, | |
| "usage:captured": 0, | |
| "usage:captured_in_exchange": 60, | |
| "x_request_id_header:applicable": 60, | |
| "x_request_id_header:captured": 0, | |
| "x_request_id_header:captured_in_exchange": 60 | |
| }, | |
| "geval_steps": { | |
| "exact_response_body:applicable": 60, | |
| "exact_response_body:captured": 0, | |
| "exact_response_body:captured_in_exchange": 60, | |
| "exchange_present": 60, | |
| "exchange_status_live": 60, | |
| "finish_reason:applicable": 60, | |
| "finish_reason:captured": 0, | |
| "finish_reason:captured_in_exchange": 60, | |
| "identical_to_wire_content": 60, | |
| "model:applicable": 60, | |
| "model:captured": 0, | |
| "model:captured_in_exchange": 60, | |
| "pairs": 60, | |
| "provider_request_id:applicable": 60, | |
| "provider_request_id:captured": 0, | |
| "provider_request_id:captured_in_exchange": 60, | |
| "response_body_sha256:applicable": 60, | |
| "response_body_sha256:captured": 0, | |
| "response_body_sha256:captured_in_exchange": 60, | |
| "response_headers:applicable": 60, | |
| "response_headers:captured": 0, | |
| "response_headers:captured_in_exchange": 0, | |
| "round_trips": 60, | |
| "system_fingerprint:applicable": 60, | |
| "system_fingerprint:captured": 0, | |
| "system_fingerprint:captured_in_exchange": 60, | |
| "usage:applicable": 60, | |
| "usage:captured": 0, | |
| "usage:captured_in_exchange": 60, | |
| "x_request_id_header:applicable": 60, | |
| "x_request_id_header:captured": 0, | |
| "x_request_id_header:captured_in_exchange": 60 | |
| }, | |
| "tc_extract": { | |
| "exact_response_body:applicable": 60, | |
| "exact_response_body:captured": 0, | |
| "exact_response_body:captured_in_exchange": 60, | |
| "exchange_present": 60, | |
| "exchange_status_live": 60, | |
| "finish_reason:applicable": 60, | |
| "finish_reason:captured": 0, | |
| "finish_reason:captured_in_exchange": 60, | |
| "identical_to_wire_content": 60, | |
| "model:applicable": 60, | |
| "model:captured": 0, | |
| "model:captured_in_exchange": 60, | |
| "pairs": 60, | |
| "provider_request_id:applicable": 60, | |
| "provider_request_id:captured": 0, | |
| "provider_request_id:captured_in_exchange": 60, | |
| "response_body_sha256:applicable": 60, | |
| "response_body_sha256:captured": 0, | |
| "response_body_sha256:captured_in_exchange": 60, | |
| "response_headers:applicable": 60, | |
| "response_headers:captured": 0, | |
| "response_headers:captured_in_exchange": 0, | |
| "round_trips": 60, | |
| "system_fingerprint:applicable": 60, | |
| "system_fingerprint:captured": 0, | |
| "system_fingerprint:captured_in_exchange": 60, | |
| "usage:applicable": 60, | |
| "usage:captured": 0, | |
| "usage:captured_in_exchange": 60, | |
| "x_request_id_header:applicable": 60, | |
| "x_request_id_header:captured": 0, | |
| "x_request_id_header:captured_in_exchange": 60 | |
| }, | |
| "tc_verdict": { | |
| "exact_response_body:applicable": 60, | |
| "exact_response_body:captured": 0, | |
| "exact_response_body:captured_in_exchange": 60, | |
| "exchange_present": 60, | |
| "exchange_status_live": 60, | |
| "finish_reason:applicable": 60, | |
| "finish_reason:captured": 0, | |
| "finish_reason:captured_in_exchange": 60, | |
| "identical_to_wire_content": 60, | |
| "model:applicable": 60, | |
| "model:captured": 0, | |
| "model:captured_in_exchange": 60, | |
| "pairs": 60, | |
| "provider_request_id:applicable": 60, | |
| "provider_request_id:captured": 0, | |
| "provider_request_id:captured_in_exchange": 60, | |
| "response_body_sha256:applicable": 60, | |
| "response_body_sha256:captured": 0, | |
| "response_body_sha256:captured_in_exchange": 60, | |
| "response_headers:applicable": 60, | |
| "response_headers:captured": 0, | |
| "response_headers:captured_in_exchange": 0, | |
| "round_trips": 60, | |
| "system_fingerprint:applicable": 60, | |
| "system_fingerprint:captured": 0, | |
| "system_fingerprint:captured_in_exchange": 60, | |
| "usage:applicable": 60, | |
| "usage:captured": 0, | |
| "usage:captured_in_exchange": 60, | |
| "x_request_id_header:applicable": 60, | |
| "x_request_id_header:captured": 0, | |
| "x_request_id_header:captured_in_exchange": 60 | |
| } | |
| }, | |
| "M3d_score_recomputable": { | |
| "geval": { | |
| "recomputable": 37, | |
| "recomputable_from_archive": 60, | |
| "units": 60 | |
| }, | |
| "task_completion": { | |
| "recomputable": 60, | |
| "recomputable_from_archive": 60, | |
| "units": 60 | |
| } | |
| }, | |
| "M4_cache": { | |
| "cache:cold": { | |
| "fork_capture_status": "live", | |
| "fork_captures": 2, | |
| "m4_rule_in_capture": 0, | |
| "m4_rule_on_wire": 0, | |
| "metricdata_capture_status": "live", | |
| "metricdata_captures": 2, | |
| "metricdata_exchange_statuses": [ | |
| "live" | |
| ], | |
| "score": 1.0, | |
| "wire_calls": 2 | |
| }, | |
| "cache:warm": { | |
| "fork_capture_status": "replay", | |
| "fork_captures": 2, | |
| "m4_rule_in_capture": 0, | |
| "m4_rule_on_wire": 0, | |
| "metricdata_capture_status": "replay", | |
| "metricdata_captures": 2, | |
| "metricdata_exchange_statuses": [ | |
| "replay" | |
| ], | |
| "score": 1.0, | |
| "wire_calls": 0 | |
| }, | |
| "cache:warm_mutated": { | |
| "fork_capture_status": "live", | |
| "fork_captures": 2, | |
| "m4_rule_in_capture": 1, | |
| "m4_rule_on_wire": 1, | |
| "metricdata_capture_status": "live", | |
| "metricdata_captures": 2, | |
| "metricdata_exchange_statuses": [ | |
| "live" | |
| ], | |
| "score": 1.0, | |
| "wire_calls": 2 | |
| }, | |
| "metricdata_fields_differing_cold_vs_warm": [ | |
| "evaluation_cost", | |
| "metricdata_judge_capture_status" | |
| ], | |
| "warm_captures_equal_cold": true | |
| }, | |
| "M5_task_inference": { | |
| "score_differs": 14, | |
| "task_text_differs": 13, | |
| "traces": 20 | |
| }, | |
| "calls_reserved": 244, | |
| "exploratory_truth_agreement": { | |
| "geval": { | |
| "truth0_pass0": 28, | |
| "truth0_pass1": 2, | |
| "truth1_pass0": 21, | |
| "truth1_pass1": 9 | |
| }, | |
| "task_completion": { | |
| "truth0_pass0": 9, | |
| "truth0_pass1": 21, | |
| "truth1_pass0": 10, | |
| "truth1_pass1": 20 | |
| } | |
| }, | |
| "finish_reason_length": 0, | |
| "integrity_failures": [], | |
| "mode": "live", | |
| "prereg_chain": "current", | |
| "reportable": true, | |
| "run_identity_problems": [], | |
| "served_models": { | |
| "gpt-4.1-mini-2025-04-14": 240 | |
| }, | |
| "usage_priced_usd": 0.08439, | |
| "usd_reserved": 0.506698 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment