{ "schema_version": 2, "project": "compaction-check", "subset": "full", "repeat": 3, "timestamp": "2026-09-24T16:34:45Z", "input_hashes": { "samples/compaction-check/fixtures/questions.json": "0d348f75b58e9df36b14babdfe9ae57f192a70d2452d757385839706712f00d1", "samples/compaction-check/fixtures/sessions.jsonl": "6b6e01855f16358971ca6734810ebec8d3a7b2240d9a8c05dbfd7c3f0ca115e6" }, "source_hashes": { "samples/_shared/community.py": "88155b9fcb87dbc2af50a59a02cdcc85a87e9404debaa6e6e3a96744c104d2cd", "samples/_shared/jev.py": "5a0a37f5097a0ba9c7ccfc4a75990c7d825c809e48f2e407341526b6b8bbcc28", "samples/compaction-check/build_excerpts.py": "28a991f6ddcd8324b2336543bfde6f3d3ba379dd78c45a876d24d93d54d1e981", "samples/compaction-check/compact.py": "80df701cac548bf404ad02794635baab5bb7cf92b2b9e7acd92c263b6c9fd5ec" }, "label": "openjev (Codiv free hosted) - NOT JEV", "mode": "live", "provider": "codiv", "rows": [ { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 3, "method": "full", "context_chars": 16157, "tokens_estimate": 4040, "tokens_full_estimate": 4040, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "1": "keep", "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "16": "keep", "18": "keep", "21": "keep", "23": "keep", "25": "keep", "27": "keep", "30": "keep", "31": "keep", "33": "keep", "35": "keep", "37": "keep", "41": "keep", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7849, "output_tokens": 0, "median_latency_ms": 2304.7, "wall_s": 2.305, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 3, "method": "last6", "context_chars": 12139, "tokens_estimate": 3035, "tokens_full_estimate": 4040, "tokens_saved_estimate": 1005, "saved_fraction": 0.24876237623762376, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "1": "drop", "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "drop", "14": "drop", "16": "drop", "18": "drop", "21": "drop", "23": "drop", "25": "drop", "27": "drop", "30": "drop", "31": "drop", "33": "drop", "35": "drop", "37": "drop", "41": "drop", "43": "drop", "45": "drop", "47": "drop", "49": "drop", "51": "drop", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5447, "output_tokens": 0, "median_latency_ms": 1142.3, "wall_s": 1.143, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 3, "method": "trunc120", "context_chars": 12682, "tokens_estimate": 3171, "tokens_full_estimate": 4040, "tokens_saved_estimate": 869, "saved_fraction": 0.2150990099009901, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "1": "truncate", "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "truncate", "14": "truncate", "16": "truncate", "18": "truncate", "21": "truncate", "23": "truncate", "25": "truncate", "27": "truncate", "30": "truncate", "31": "truncate", "33": "truncate", "35": "truncate", "37": "truncate", "41": "truncate", "43": "truncate", "45": "truncate", "47": "truncate", "49": "truncate", "51": "truncate", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5936, "output_tokens": 0, "median_latency_ms": 1101.4, "wall_s": 1.954, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 3, "method": "model", "context_chars": 15157, "tokens_estimate": 3790, "tokens_full_estimate": 4040, "tokens_saved_estimate": 250, "saved_fraction": 0.06188118811881188, "decisions": { "keep": 21, "truncate": 0, "drop": 9 }, "result_decisions": { "1": "keep", "3": "drop", "5": "keep", "7": "keep", "9": "keep", "12": "drop", "14": "keep", "16": "keep", "18": "keep", "21": "drop", "23": "keep", "25": "drop", "27": "keep", "30": "drop", "31": "drop", "33": "keep", "35": "keep", "37": "keep", "41": "drop", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "drop", "60": "keep", "62": "keep", "64": "drop" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": false, "gold": "95586", "answer": "95590", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 25163, "output_tokens": 0, "median_latency_ms": 2027.1, "wall_s": 4.951, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 18054, "output_tokens": 0, "median_latency_ms": 2806.7, "wall_s": 3.703, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 8113, "selector_reuse_break_even_estimated_reads": 33 }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 3, "method": "full", "context_chars": 9094, "tokens_estimate": 2274, "tokens_full_estimate": 2274, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 10, "truncate": 0, "drop": 0 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4332, "output_tokens": 0, "median_latency_ms": 751.0, "wall_s": 1.5, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 3, "method": "last6", "context_chars": 8362, "tokens_estimate": 2091, "tokens_full_estimate": 2274, "tokens_saved_estimate": 183, "saved_fraction": 0.08047493403693931, "decisions": { "keep": 6, "truncate": 0, "drop": 4 }, "result_decisions": { "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27638", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 3978, "output_tokens": 0, "median_latency_ms": 863.2, "wall_s": 2.108, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 3, "method": "trunc120", "context_chars": 8718, "tokens_estimate": 2180, "tokens_full_estimate": 2274, "tokens_saved_estimate": 94, "saved_fraction": 0.04133685136323659, "decisions": { "keep": 6, "truncate": 4, "drop": 0 }, "result_decisions": { "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4144, "output_tokens": 0, "median_latency_ms": 640.7, "wall_s": 1.77, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 3, "method": "model", "context_chars": 8897, "tokens_estimate": 2225, "tokens_full_estimate": 2274, "tokens_saved_estimate": 49, "saved_fraction": 0.021547933157431837, "decisions": { "keep": 5, "truncate": 0, "drop": 5 }, "result_decisions": { "3": "keep", "5": "keep", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "drop", "19": "drop", "22": "keep", "24": "drop" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 8852, "output_tokens": 0, "median_latency_ms": 848.9, "wall_s": 4.099, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4635, "output_tokens": 0, "median_latency_ms": 947.5, "wall_s": 2.3, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 3784, "selector_reuse_break_even_estimated_reads": 78 }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 3, "method": "full", "context_chars": 19813, "tokens_estimate": 4954, "tokens_full_estimate": 4954, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "4": "keep", "6": "keep", "8": "keep", "10": "keep", "13": "keep", "15": "keep", "17": "keep", "19": "keep", "22": "keep", "23": "keep", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "keep", "36": "keep", "39": "keep", "41": "keep", "45": "keep", "46": "keep", "48": "keep", "51": "keep", "52": "keep", "54": "keep", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 9347, "output_tokens": 0, "median_latency_ms": 1471.6, "wall_s": 2.714, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 3, "method": "last6", "context_chars": 15312, "tokens_estimate": 3828, "tokens_full_estimate": 4954, "tokens_saved_estimate": 1126, "saved_fraction": 0.2272910779168349, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "4": "drop", "6": "drop", "8": "drop", "10": "drop", "13": "drop", "15": "drop", "17": "drop", "19": "drop", "22": "drop", "23": "drop", "26": "drop", "28": "drop", "30": "drop", "32": "drop", "34": "drop", "36": "drop", "39": "drop", "41": "drop", "45": "drop", "46": "drop", "48": "drop", "51": "drop", "52": "drop", "54": "drop", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 2, "correct": 2, "abstained": 2, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 3, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 1, "correct": 1, "abstained": 2, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 3 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "yes", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": false, "gold": "err", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds3", "kind": "tool_dependent", "retained": false, "gold": "1.364", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds4", "kind": "tool_dependent", "retained": false, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 6599, "output_tokens": 0, "median_latency_ms": 1382.2, "wall_s": 1.903, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 3, "method": "trunc120", "context_chars": 17221, "tokens_estimate": 4306, "tokens_full_estimate": 4954, "tokens_saved_estimate": 648, "saved_fraction": 0.1308033911990311, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "4": "truncate", "6": "truncate", "8": "truncate", "10": "truncate", "13": "truncate", "15": "truncate", "17": "truncate", "19": "truncate", "22": "truncate", "23": "truncate", "26": "truncate", "28": "truncate", "30": "truncate", "32": "truncate", "34": "truncate", "36": "truncate", "39": "truncate", "41": "truncate", "45": "truncate", "46": "truncate", "48": "truncate", "51": "truncate", "52": "truncate", "54": "truncate", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7675, "output_tokens": 0, "median_latency_ms": 1341.3, "wall_s": 1.941, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 3, "method": "model", "context_chars": 18386, "tokens_estimate": 4597, "tokens_full_estimate": 4954, "tokens_saved_estimate": 357, "saved_fraction": 0.07206297941057731, "decisions": { "keep": 20, "truncate": 0, "drop": 10 }, "result_decisions": { "4": "keep", "6": "keep", "8": "drop", "10": "keep", "13": "keep", "15": "keep", "17": "drop", "19": "keep", "22": "drop", "23": "keep", "26": "keep", "28": "drop", "30": "keep", "32": "keep", "34": "drop", "36": "keep", "39": "keep", "41": "keep", "45": "drop", "46": "drop", "48": "keep", "51": "keep", "52": "drop", "54": "keep", "56": "drop", "59": "keep", "61": "keep", "63": "drop", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "52430", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 29537, "output_tokens": 0, "median_latency_ms": 2658.3, "wall_s": 5.968, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 20974, "output_tokens": 0, "median_latency_ms": 4006.4, "wall_s": 4.657, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 9097, "selector_reuse_break_even_estimated_reads": 26 } ], "summary": { "methods": { "full": { "sessions": 3, "tokens_estimate": 11268, "tokens_full_estimate": 11268, "saved_fraction": 0.0, "questions": 17, "answered": 17, "retained": 17, "correct": 15, "abstained": 0, "answerability_correct": 15, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 11, "correct": 10, "abstained": 0, "answerability_correct": 10, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "last6": { "sessions": 3, "tokens_estimate": 8954, "tokens_full_estimate": 11268, "saved_fraction": 0.20536031238906638, "questions": 17, "answered": 17, "retained": 11, "correct": 11, "abstained": 3, "answerability_correct": 13, "unsupported_correct_guess": 1, "lost_answer": 5, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 5, "correct": 5, "abstained": 3, "answerability_correct": 7, "unsupported_correct_guess": 1, "lost_answer": 5 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "trunc120": { "sessions": 3, "tokens_estimate": 9657, "tokens_full_estimate": 11268, "saved_fraction": 0.14297124600638977, "questions": 17, "answered": 17, "retained": 15, "correct": 15, "abstained": 1, "answerability_correct": 15, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 9, "correct": 9, "abstained": 1, "answerability_correct": 9, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "model": { "sessions": 3, "tokens_estimate": 10612, "tokens_full_estimate": 11268, "saved_fraction": 0.058217962371317045, "questions": 17, "answered": 17, "retained": 16, "correct": 15, "abstained": 0, "answerability_correct": 15, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 10, "correct": 10, "abstained": 0, "answerability_correct": 10, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } } } }, "policy": { "version": "compaction-v2", "methods": [ "full", "last6", "trunc120", "model" ], "keep_last_results": 6, "truncate_chars": 120, "model_decisions": [ "keep", "drop" ], "token_estimate": "ceil(rendered context characters / 4); not provider token usage", "association": "singleton balanced visible group with no history gap; otherwise unknown", "evidence": "any annotated alternative whose event-specific spans all survive", "scope": "three committed scrubbed excerpts; no full-transcript extrapolation" }, "status": "complete", "calls": [ { "case_id": "claude-code__opus-5-5:repeat-3", "phase": "reader:full", "state_hash": "09f53da178caf35ec8ab71f38ae731fb32bee8d62f2b4f473e617cde73e00f1d", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.998019529385399, "probabilities": { "yes": 9.7113337925116e-05, "no": 0.9997274768079372, "unknown": 1.8324768240436994e-05, "insufficient_evidence": 0.00015708508589716767 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9777448793835319, "probabilities": { "commons": 0.9951311393127591, "sony": 0.00024677298476438855, "none": 0.002849010702014509, "generated": 0.00042923452841309894, "insufficient_evidence": 0.0013438424720489711 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.5275270515676409, "probabilities": { "209394": 0.6492946747920596, "223434": 0.005248056602942934, "233659": 0.01614649012738712, "379231": 0.3256637171812993, "insufficient_evidence": 0.0036470612963110357 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.8403937047953001, "probabilities": { "95586": 0.9434891415984152, "95590": 0.04285614427435183, "47158": 0.0012710424808341301, "27638": 0.01109094494830993, "insufficient_evidence": 0.0012927266980889386 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9832178689176051, "probabilities": { "groups": 0.9964609399478898, "none": 0.0020514846184676943, "gloss": 0.0010459127132880098, "all": 0.0003733774164554525, "insufficient_evidence": 6.828530389911293e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.9314261907041628, "probabilities": { "yes": 0.006614247537742373, "no": 0.9875982152146019, "insufficient_evidence": 0.005787537247655682 } } }, "usage": { "input_tokens": 7849, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 2304.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.305 }, { "case_id": "claude-code__opus-5-5:repeat-3", "phase": "reader:last6", "state_hash": "d92967ac2586f77e2370c343f2d044ea4cab43cbed4ae6082644954af3502493", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.978840432753249, "probabilities": { "yes": 0.002246004135292316, "no": 0.9960421573237017, "unknown": 0.00029149615254747793, "insufficient_evidence": 0.0014203423884584956 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9887306554619513, "probabilities": { "commons": 0.9977622831150538, "sony": 0.0002245420123020538, "none": 0.0005878246627847546, "generated": 0.0013475284304472201, "insufficient_evidence": 7.782177941223092e-05 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9196439367906956, "probabilities": { "209394": 0.0002489668691454731, "223434": 0.003658231084558018, "233659": 0.011584681820168452, "379231": 0.006621755087417425, "insufficient_evidence": 0.9778863651387106 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.8985253260261111, "probabilities": { "95586": 0.969092507832787, "95590": 0.018090581148993253, "47158": 0.0007760910756695681, "27638": 0.011704557208141099, "insufficient_evidence": 0.00033626273440904057 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9855961463100867, "probabilities": { "groups": 0.9970898894312566, "none": 0.0015162617938270187, "gloss": 0.0005540175662069775, "all": 0.0007270565288480297, "insufficient_evidence": 0.00011277467986135785 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.3731404281476718, "probabilities": { "yes": 0.719484968578665, "no": 0.2505911740165728, "insufficient_evidence": 0.02992385740476225 } } }, "usage": { "input_tokens": 5447, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1142.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.143 }, { "case_id": "claude-code__opus-5-5:repeat-3", "phase": "reader:trunc120", "state_hash": "308d7e00b7db233c2c64067c428d6533c53b57192d415d967f011d0d6a1a30df", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9708656940568073, "probabilities": { "yes": 0.002990651500468079, "no": 0.9943756980139282, "unknown": 0.0016189902745408747, "insufficient_evidence": 0.0010146602110628648 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.992247815416466, "probabilities": { "commons": 0.9985932214711324, "sony": 0.00023172857374989672, "none": 0.000568064481206864, "generated": 0.00039913668325540036, "insufficient_evidence": 0.00020784879065541536 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.8855949783667808, "probabilities": { "209394": 0.0006514693254419826, "223434": 0.008983224058952514, "233659": 0.014023529638817938, "379231": 0.009573292680947691, "insufficient_evidence": 0.9667684842958398 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.695934930678596, "probabilities": { "95586": 0.8492165452104469, "95590": 0.010638206898757855, "47158": 0.002344089789331352, "27638": 0.13467268198398802, "insufficient_evidence": 0.0031284761174758826 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9255030893434641, "probabilities": { "groups": 0.9781429891649502, "none": 0.0036387738132964706, "gloss": 0.01701578859262618, "all": 0.0010558737773427092, "insufficient_evidence": 0.0001465746517844208 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.6697483792905067, "probabilities": { "yes": 0.9056200649371148, "no": 0.07319083036893036, "insufficient_evidence": 0.02118910469395492 } } }, "usage": { "input_tokens": 5936, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1101.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.954 }, { "case_id": "claude-code__opus-5-5:repeat-3", "phase": "select", "state_hash": "1ab156da003911d9ea375ce54aacbf19201cc6adba8bc73315f688d4ed39937a", "questions_hash": "b934e72498fb1f126d8c79bf9a0c69e5e573708edd8658a1f5e2ae40d4d4cb83", "question_ids": [ "result_1", "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_16", "result_18", "result_21", "result_23", "result_25", "result_27", "result_30", "result_31", "result_33", "result_35", "result_37", "result_41", "result_43", "result_45", "result_47", "result_49", "result_51", "result_53", "result_56", "result_58", "result_60", "result_62", "result_64" ], "answers": { "result_1": { "type": "choice", "choice": "keep", "confidence": 0.8562832219790479, "probabilities": { "keep": 0.9795935183603788, "drop": 0.020406481639621145 } }, "result_3": { "type": "choice", "choice": "drop", "confidence": 0.9392628780102992, "probabilities": { "keep": 0.007079085518264908, "drop": 0.992920914481735 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.7491172264354212, "probabilities": { "keep": 0.9581119669657476, "drop": 0.04188803303425242 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.2288883272242589, "probabilities": { "keep": 0.7739218867161817, "drop": 0.22607811328381827 } }, "result_9": { "type": "choice", "choice": "keep", "confidence": 0.7701547380127299, "probabilities": { "keep": 0.9626845460175788, "drop": 0.03731545398242125 } }, "result_12": { "type": "choice", "choice": "drop", "confidence": 0.09160386805081622, "probabilities": { "keep": 0.32373420139918013, "drop": 0.6762657986008198 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.07808864323527831, "probabilities": { "keep": 0.663007765951238, "drop": 0.336992234048762 } }, "result_16": { "type": "choice", "choice": "keep", "confidence": 0.0001900576549530575, "probabilities": { "keep": 0.5081157914774936, "drop": 0.49188420852250636 } }, "result_18": { "type": "choice", "choice": "keep", "confidence": 0.49025123911478785, "probabilities": { "keep": 0.886713962465564, "drop": 0.11328603753443604 } }, "result_21": { "type": "choice", "choice": "drop", "confidence": 0.4624463031356403, "probabilities": { "keep": 0.122869643211262, "drop": 0.877130356788738 } }, "result_23": { "type": "choice", "choice": "keep", "confidence": 0.5088929069731932, "probabilities": { "keep": 0.89289971874911, "drop": 0.10710028125089 } }, "result_25": { "type": "choice", "choice": "drop", "confidence": 0.012448914483622064, "probabilities": { "keep": 0.4344100549062482, "drop": 0.5655899450937518 } }, "result_27": { "type": "choice", "choice": "keep", "confidence": 0.1514421804520263, "probabilities": { "keep": 0.7249931061244826, "drop": 0.27500689387551736 } }, "result_30": { "type": "choice", "choice": "drop", "confidence": 0.09062358597870301, "probabilities": { "keep": 0.3246592276999844, "drop": 0.6753407723000155 } }, "result_31": { "type": "choice", "choice": "drop", "confidence": 0.15073920274794683, "probabilities": { "keep": 0.27551021469389886, "drop": 0.7244897853061012 } }, "result_33": { "type": "choice", "choice": "keep", "confidence": 0.04108222568566411, "probabilities": { "keep": 0.618753294955889, "drop": 0.381246705044111 } }, "result_35": { "type": "choice", "choice": "keep", "confidence": 0.3169231981176386, "probabilities": { "keep": 0.8186245444088724, "drop": 0.18137545559112747 } }, "result_37": { "type": "choice", "choice": "keep", "confidence": 0.23674379508639587, "probabilities": { "keep": 0.7783018335397478, "drop": 0.22169816646025226 } }, "result_41": { "type": "choice", "choice": "drop", "confidence": 0.010526883406897003, "probabilities": { "keep": 0.4396720834331318, "drop": 0.5603279165668682 } }, "result_43": { "type": "choice", "choice": "keep", "confidence": 0.06974479584348314, "probabilities": { "keep": 0.6542063693564161, "drop": 0.3457936306435839 } }, "result_45": { "type": "choice", "choice": "keep", "confidence": 0.00523815359763824, "probabilities": { "keep": 0.5425817742809758, "drop": 0.45741822571902424 } }, "result_47": { "type": "choice", "choice": "keep", "confidence": 0.08435298710429584, "probabilities": { "keep": 0.6692930669680399, "drop": 0.3307069330319601 } }, "result_49": { "type": "choice", "choice": "keep", "confidence": 0.013061169223767766, "probabilities": { "keep": 0.5671787118102675, "drop": 0.43282128818973253 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.06640389727337603, "probabilities": { "keep": 0.6505274642256742, "drop": 0.3494725357743258 } }, "result_53": { "type": "choice", "choice": "keep", "confidence": 0.007532082251421324, "probabilities": { "keep": 0.5510477693096193, "drop": 0.44895223069038076 } }, "result_56": { "type": "choice", "choice": "keep", "confidence": 0.028791726125425132, "probabilities": { "keep": 0.5995584926752727, "drop": 0.4004415073247273 } }, "result_58": { "type": "choice", "choice": "drop", "confidence": 2.935966247108457e-05, "probabilities": { "keep": 0.49681013877926, "drop": 0.50318986122074 } }, "result_60": { "type": "choice", "choice": "keep", "confidence": 0.0022413775374127143, "probabilities": { "keep": 0.5278639533400705, "drop": 0.47213604665992953 } }, "result_62": { "type": "choice", "choice": "keep", "confidence": 0.11110737773158985, "probabilities": { "keep": 0.6936690928555991, "drop": 0.3063309071444009 } }, "result_64": { "type": "choice", "choice": "drop", "confidence": 0.02082331605246457, "probabilities": { "keep": 0.4152531863351339, "drop": 0.5847468136648661 } } }, "usage": { "input_tokens": 18054, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 2806.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 3.703 }, { "case_id": "claude-code__opus-5-5:repeat-3", "phase": "reader:model", "state_hash": "31367735d4404e4cd99e36320c8e918b263fd3170aabe247707bffdc85c95490", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9953627215039962, "probabilities": { "yes": 0.0005729210597170295, "no": 0.9992751985357108, "unknown": 4.4636397113773517e-05, "insufficient_evidence": 0.00010724400745836813 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9375327408879717, "probabilities": { "commons": 0.9833153594547586, "sony": 0.0003680683668161939, "none": 0.009617302400578953, "generated": 0.0010344386735043078, "insufficient_evidence": 0.005664831104341892 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.7751547053504207, "probabilities": { "209394": 0.9151127884208826, "223434": 0.004523859182841699, "233659": 0.017215926512039036, "379231": 0.06019855525528353, "insufficient_evidence": 0.0029488706289531627 } }, "cc4": { "type": "choice", "choice": "95590", "confidence": 0.6040377755583095, "probabilities": { "95586": 0.18363493820128535, "95590": 0.7856329788118928, "47158": 0.0049325334382639575, "27638": 0.017836474250627052, "insufficient_evidence": 0.007963075297930885 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9834379486027723, "probabilities": { "groups": 0.9964597055328344, "none": 0.0024223675316966346, "gloss": 0.00047652049242713886, "all": 0.0005931649323957943, "insufficient_evidence": 4.824151064610816e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.35768368925358396, "probabilities": { "yes": 0.11821751800946141, "no": 0.7663129381658427, "insufficient_evidence": 0.11546954382469593 } } }, "usage": { "input_tokens": 7109, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1247.6, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.248 }, { "case_id": "codex__gpt-6-astra:repeat-3", "phase": "reader:full", "state_hash": "818641e42779c98df892775aec9b311b7dd75a0f45931036724b66d247b82d59", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9992662043409002, "probabilities": { "reference": 0.9999026527869669, "render": 9.282414928716647e-05, "none": 1.0068300222202873e-06, "insufficient_evidence": 3.5162337238163762e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9996388332214251, "probabilities": { "download": 0.9999580410647053, "drawn": 2.599175113393308e-05, "omitted": 3.167793428207335e-06, "insufficient_evidence": 1.279939073252296e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9828434354893076, "probabilities": { "script": 0.996710888722046, "patch": 0.00040241296239835035, "echo": 0.0027746456247670723, "insufficient_evidence": 0.0001120526907884054 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.9810490988486702, "probabilities": { "27616": 0.9952916068089098, "27638": 0.004606109095727534, "47158": 1.6196471991171497e-05, "95586": 7.34112588886008e-05, "insufficient_evidence": 1.2676364483017192e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9970885153763517, "probabilities": { "valid": 0.9995897880181163, "invalid": 0.0001929518106590194, "notrun": 8.055449613490023e-05, "insufficient_evidence": 0.00013670567508984778 } } }, "usage": { "input_tokens": 4332, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 751.0, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.5 }, { "case_id": "codex__gpt-6-astra:repeat-3", "phase": "reader:last6", "state_hash": "37c153d6143d560dec7b586528ae5881e640484c9fb38cbdb6b22ac0ed8ce18b", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9991342642014704, "probabilities": { "reference": 0.9998825748789351, "render": 0.00011329304754642146, "none": 9.22836325999979e-07, "insufficient_evidence": 3.2092371925311214e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9991615413222976, "probabilities": { "download": 0.9998939539344831, "drawn": 2.6702773392699993e-05, "omitted": 8.01610735343772e-06, "insufficient_evidence": 7.13271847707551e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9541696891902112, "probabilities": { "script": 0.9890235138134467, "patch": 0.0004265817519742824, "echo": 0.010270907688980514, "insufficient_evidence": 0.00027899674559854633 } }, "cx4": { "type": "choice", "choice": "27638", "confidence": 0.9403256832721956, "probabilities": { "27616": 0.016945324984715335, "27638": 0.9819083510706683, "47158": 0.00034756106942203407, "95586": 0.0004332981828202048, "insufficient_evidence": 0.00036546469237407644 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9950915875586764, "probabilities": { "valid": 0.9992373074332893, "invalid": 0.0005680432319608514, "notrun": 8.686121845065082e-05, "insufficient_evidence": 0.00010778811629924232 } } }, "usage": { "input_tokens": 3978, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 863.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.108 }, { "case_id": "codex__gpt-6-astra:repeat-3", "phase": "reader:trunc120", "state_hash": "f3efcd2121b75907229d1975779876597872e4ff72d0dd63699692848b9fae3f", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9991816524548738, "probabilities": { "reference": 0.9998901448822897, "render": 0.00010495979825639166, "none": 1.6532961734695012e-06, "insufficient_evidence": 3.242023280533377e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9994578433625599, "probabilities": { "download": 0.9999338098471637, "drawn": 1.9628410610975636e-05, "omitted": 1.988736943881972e-06, "insufficient_evidence": 4.457300528129796e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9771030839771728, "probabilities": { "script": 0.9952812878218108, "patch": 0.000341850262254844, "echo": 0.004261479252405555, "insufficient_evidence": 0.00011538266352890883 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.975444922222406, "probabilities": { "27616": 0.993874577194587, "27638": 0.005559240959673349, "47158": 2.7588296044341996e-05, "95586": 0.0004703741511808017, "insufficient_evidence": 6.82193985144197e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9990527809570378, "probabilities": { "valid": 0.9998790586889565, "invalid": 7.197599451089132e-05, "notrun": 6.75542462186418e-06, "insufficient_evidence": 4.220989191078784e-05 } } }, "usage": { "input_tokens": 4144, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 640.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.77 }, { "case_id": "codex__gpt-6-astra:repeat-3", "phase": "select", "state_hash": "f3b419d113ea7359abd8053f120ba9422237f5cf245aba011093c383b4fc09ec", "questions_hash": "9cac56873501140b02d037d69f22e218df2321487095362f67a020b3f35c89ef", "question_ids": [ "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_17", "result_19", "result_22", "result_24" ], "answers": { "result_3": { "type": "choice", "choice": "keep", "confidence": 0.10798835525688333, "probabilities": { "keep": 0.6910035407464342, "drop": 0.3089964592535659 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.031112991180994753, "probabilities": { "keep": 0.6034659398752724, "drop": 0.3965340601247275 } }, "result_7": { "type": "choice", "choice": "drop", "confidence": 0.009322400723655866, "probabilities": { "keep": 0.4432203140574239, "drop": 0.5567796859425761 } }, "result_9": { "type": "choice", "choice": "drop", "confidence": 0.33957828228070985, "probabilities": { "keep": 0.17119085653715935, "drop": 0.8288091434628406 } }, "result_12": { "type": "choice", "choice": "keep", "confidence": 0.0006294069353327547, "probabilities": { "keep": 0.5147683478196455, "drop": 0.4852316521803544 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.2387537937073385, "probabilities": { "keep": 0.779408438759746, "drop": 0.22059156124025406 } }, "result_17": { "type": "choice", "choice": "drop", "confidence": 3.390168655414971e-05, "probabilities": { "keep": 0.496572269381552, "drop": 0.5034277306184479 } }, "result_19": { "type": "choice", "choice": "drop", "confidence": 0.5738005275361586, "probabilities": { "keep": 0.08694791881133163, "drop": 0.9130520811886684 } }, "result_22": { "type": "choice", "choice": "keep", "confidence": 0.6365902163649351, "probabilities": { "keep": 0.9306722872738571, "drop": 0.06932771272614291 } }, "result_24": { "type": "choice", "choice": "drop", "confidence": 0.0014059282277567586, "probabilities": { "keep": 0.4779296739259129, "drop": 0.5220703260740871 } } }, "usage": { "input_tokens": 4635, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 947.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.3 }, { "case_id": "codex__gpt-6-astra:repeat-3", "phase": "reader:model", "state_hash": "20db1b83b30abee5a087b9cf6c4582474e63b501ee26914a393e7612f5455f21", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9991259507543067, "probabilities": { "reference": 0.9998810484374631, "render": 0.00011541769890246591, "none": 8.05770641762062e-07, "insufficient_evidence": 2.7280929927034073e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9989923061590757, "probabilities": { "download": 0.99987114402064, "drawn": 6.429782887036942e-05, "omitted": 8.242456106691752e-06, "insufficient_evidence": 5.631569438284663e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9855512231429044, "probabilities": { "script": 0.9974829581144794, "patch": 0.0006912313207626548, "echo": 0.0014152098965198223, "insufficient_evidence": 0.00041060066823810385 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.5759674096664313, "probabilities": { "27616": 0.5986828686395995, "27638": 0.4001735948400727, "47158": 0.0001314275222275719, "95586": 0.00036270836163450186, "insufficient_evidence": 0.0006494006364657315 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9969597126718561, "probabilities": { "valid": 0.9995665070320594, "invalid": 0.00024374940333923584, "notrun": 7.522329939561413e-05, "insufficient_evidence": 0.00011452026520578116 } } }, "usage": { "input_tokens": 4217, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 750.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.799 }, { "case_id": "dsh__deepseek-flash:repeat-3", "phase": "reader:full", "state_hash": "2c718871c891948a99e03c50664a5cacbfa33a885c063e9ae0760a8b166fb44a", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9993700484547665, "probabilities": { "yes": 3.0705274450126753e-05, "auth": 0.9999227380175028, "timeout": 3.5640232033417614e-06, "insufficient_evidence": 4.299268484369815e-05 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.9961773090599801, "probabilities": { "err": 0.9994329456600654, "ok": 0.00011611971314308998, "missing": 7.708449837323984e-05, "insufficient_evidence": 0.0003738501284182988 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9988292825383837, "probabilities": { "1.364": 0.9998238108900088, "1.000": 5.8980879474559376e-05, "1.455": 2.4722131370126757e-05, "0.733": 8.974369787646207e-05, "insufficient_evidence": 2.74240126999796e-06 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.982098565530729, "probabilities": { "52430": 0.0019951689772987327, "47158": 0.9962032615380788, "95586": 0.0002725408604874846, "27616": 0.001389116870526236, "insufficient_evidence": 0.0001399117536087616 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.9448523804669287, "probabilities": { "47158": 0.9834645185419371, "52430": 0.015539423128320629, "95586": 0.00044889699897859625, "27638": 0.0005298983034364298, "insufficient_evidence": 1.7263027327343094e-05 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.99206425529215, "probabilities": { "pearl": 0.9987379473249663, "black": 0.0006257424128172322, "edge": 0.00031815513110822853, "insufficient_evidence": 0.00031815513110822853 } } }, "usage": { "input_tokens": 9347, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1471.6, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.714 }, { "case_id": "dsh__deepseek-flash:repeat-3", "phase": "reader:last6", "state_hash": "f746af707c79a9b701f3828e24bbb0a2f7b6e6f79d063fdc225ec7a0a07a57c9", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "yes", "confidence": 0.9883271589731341, "probabilities": { "yes": 0.997943397367218, "auth": 0.0012626033088885633, "timeout": 5.384517586206333e-06, "insufficient_evidence": 0.0007886148063072578 } }, "ds2": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.7328713263653037, "probabilities": { "err": 0.007273736077861407, "ok": 0.09830412405456834, "missing": 0.0008370806401719389, "insufficient_evidence": 0.8935850592273984 } }, "ds3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.601333775993704, "probabilities": { "1.364": 0.036447534648746234, "1.000": 0.05222819800513616, "1.455": 0.014842045193222016, "0.733": 0.05481307211427799, "insufficient_evidence": 0.8416691500386175 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.6783631510265571, "probabilities": { "52430": 0.1445171030186753, "47158": 0.8354661042195771, "95586": 0.0008647770463824526, "27616": 0.017308647895169917, "insufficient_evidence": 0.0018433678201952028 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.5849594357841438, "probabilities": { "47158": 0.6498749813712316, "52430": 0.3468499284443135, "95586": 0.0002930423963172937, "27638": 0.002710102943062525, "insufficient_evidence": 0.00027194484507507414 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9699344767947571, "probabilities": { "pearl": 0.9941292326024949, "black": 0.0031514679627964405, "edge": 0.000758985011319903, "insufficient_evidence": 0.0019603144233887727 } } }, "usage": { "input_tokens": 6599, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1382.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.903 }, { "case_id": "dsh__deepseek-flash:repeat-3", "phase": "reader:trunc120", "state_hash": "637b10b060d11f04ea8ac550b6678f62ddfc5a4b1ef52ebda9f99a98a5dea062", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9958474316523603, "probabilities": { "yes": 0.00029402959508187425, "auth": 0.9993727158359785, "timeout": 1.8111947223094093e-05, "insufficient_evidence": 0.00031514262171655755 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.9804624842932839, "probabilities": { "err": 0.9962592296653061, "ok": 0.0003893399123599717, "missing": 0.002965705041438817, "insufficient_evidence": 0.0003857253808950777 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9931597278390141, "probabilities": { "1.364": 0.9987435399240999, "1.000": 0.0006856016595380727, "1.455": 0.0002996105375211518, "0.733": 0.00023835826373102322, "insufficient_evidence": 3.288961510980044e-05 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.5668431038206584, "probabilities": { "52430": 0.33114691118528355, "47158": 0.6583901921007514, "95586": 0.0040289155220701745, "27616": 0.006106127655883628, "insufficient_evidence": 0.0003278535360111715 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.9265169061843145, "probabilities": { "47158": 0.9790939152398905, "52430": 0.015120234160132695, "95586": 0.0032901964663091562, "27638": 0.0023910277995994123, "insufficient_evidence": 0.00010462633406825908 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9960971813947506, "probabilities": { "pearl": 0.9994289909932944, "black": 0.00025134405967996786, "edge": 8.346906531846842e-05, "insufficient_evidence": 0.00023619588170719662 } } }, "usage": { "input_tokens": 7675, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1341.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.941 }, { "case_id": "dsh__deepseek-flash:repeat-3", "phase": "select", "state_hash": "6d057db834e25182a5101a4d62be8256aafc4528a6356d6b34f5d0f2becfacc5", "questions_hash": "7f2e216b1bdff1cdf9c33d7a6e5296b61b3ef74b7ce862303c290a1473566dd9", "question_ids": [ "result_4", "result_6", "result_8", "result_10", "result_13", "result_15", "result_17", "result_19", "result_22", "result_23", "result_26", "result_28", "result_30", "result_32", "result_34", "result_36", "result_39", "result_41", "result_45", "result_46", "result_48", "result_51", "result_52", "result_54", "result_56", "result_59", "result_61", "result_63", "result_65", "result_67" ], "answers": { "result_4": { "type": "choice", "choice": "keep", "confidence": 0.44185913895222706, "probabilities": { "keep": 0.8697436302601504, "drop": 0.13025636973984955 } }, "result_6": { "type": "choice", "choice": "keep", "confidence": 0.018353754127330424, "probabilities": { "keep": 0.579585841084713, "drop": 0.42041415891528694 } }, "result_8": { "type": "choice", "choice": "drop", "confidence": 0.07522163611110744, "probabilities": { "keep": 0.3399578492482371, "drop": 0.660042150751763 } }, "result_10": { "type": "choice", "choice": "keep", "confidence": 0.33535152243686006, "probabilities": { "keep": 0.826943829832905, "drop": 0.17305617016709496 } }, "result_13": { "type": "choice", "choice": "keep", "confidence": 0.0005318000840450754, "probabilities": { "keep": 0.5135751691358202, "drop": 0.48642483086417976 } }, "result_15": { "type": "choice", "choice": "keep", "confidence": 0.07585107420104253, "probabilities": { "keep": 0.6606982854880178, "drop": 0.3393017145119822 } }, "result_17": { "type": "choice", "choice": "drop", "confidence": 0.0003602356950863639, "probabilities": { "keep": 0.48882691687721525, "drop": 0.5111730831227848 } }, "result_19": { "type": "choice", "choice": "keep", "confidence": 0.3718779359421496, "probabilities": { "keep": 0.842571500040515, "drop": 0.15742849995948502 } }, "result_22": { "type": "choice", "choice": "drop", "confidence": 0.018431625268832352, "probabilities": { "keep": 0.42024622759039987, "drop": 0.5797537724096001 } }, "result_23": { "type": "choice", "choice": "keep", "confidence": 0.0013968099847158078, "probabilities": { "keep": 0.5219986635412253, "drop": 0.47800133645877474 } }, "result_26": { "type": "choice", "choice": "keep", "confidence": 0.4689949121432545, "probabilities": { "keep": 0.8794272122730034, "drop": 0.12057278772699667 } }, "result_28": { "type": "choice", "choice": "drop", "confidence": 0.05243943500705761, "probabilities": { "keep": 0.36601181708575414, "drop": 0.6339881829142459 } }, "result_30": { "type": "choice", "choice": "keep", "confidence": 0.1396297558329196, "probabilities": { "keep": 0.7163547908736244, "drop": 0.2836452091263756 } }, "result_32": { "type": "choice", "choice": "keep", "confidence": 0.16680079266580472, "probabilities": { "keep": 0.7356775513060358, "drop": 0.26432244869396415 } }, "result_34": { "type": "choice", "choice": "drop", "confidence": 5.122616183450557e-05, "probabilities": { "keep": 0.49578651869654417, "drop": 0.5042134813034559 } }, "result_36": { "type": "choice", "choice": "keep", "confidence": 0.02700128817013081, "probabilities": { "keep": 0.5964334249078137, "drop": 0.4035665750921863 } }, "result_39": { "type": "choice", "choice": "keep", "confidence": 0.3964260650054151, "probabilities": { "keep": 0.8524899623770883, "drop": 0.14751003762291176 } }, "result_41": { "type": "choice", "choice": "keep", "confidence": 0.6483585386131834, "probabilities": { "keep": 0.9337839125627384, "drop": 0.0662160874372616 } }, "result_45": { "type": "choice", "choice": "drop", "confidence": 0.3436140871378005, "probabilities": { "keep": 0.1694242075032207, "drop": 0.8305757924967793 } }, "result_46": { "type": "choice", "choice": "drop", "confidence": 0.14848899637476587, "probabilities": { "keep": 0.2771302195552024, "drop": 0.7228697804447977 } }, "result_48": { "type": "choice", "choice": "keep", "confidence": 0.01265233702401769, "probabilities": { "keep": 0.5661221005628584, "drop": 0.43387789943714167 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.035944706228588474, "probabilities": { "keep": 0.6111470993709867, "drop": 0.38885290062901334 } }, "result_52": { "type": "choice", "choice": "drop", "confidence": 0.011180964555553619, "probabilities": { "keep": 0.4378308269302382, "drop": 0.5621691730697618 } }, "result_54": { "type": "choice", "choice": "keep", "confidence": 0.1824098064576024, "probabilities": { "keep": 0.7459784095813181, "drop": 0.25402159041868183 } }, "result_56": { "type": "choice", "choice": "drop", "confidence": 0.0317822567348377, "probabilities": { "keep": 0.39543535560591725, "drop": 0.6045646443940828 } }, "result_59": { "type": "choice", "choice": "keep", "confidence": 0.00023837625198297463, "probabilities": { "keep": 0.5090890243281513, "drop": 0.49091097567184866 } }, "result_61": { "type": "choice", "choice": "keep", "confidence": 0.05434743973261191, "probabilities": { "keep": 0.6363732282373459, "drop": 0.36362677176265407 } }, "result_63": { "type": "choice", "choice": "drop", "confidence": 0.00018996730575726595, "probabilities": { "keep": 0.4918861377007929, "drop": 0.5081138622992071 } }, "result_65": { "type": "choice", "choice": "keep", "confidence": 0.12732946018965863, "probabilities": { "keep": 0.706916993316734, "drop": 0.29308300668326603 } }, "result_67": { "type": "choice", "choice": "keep", "confidence": 0.020409891467676244, "probabilities": { "keep": 0.583905359559741, "drop": 0.41609464044025907 } } }, "usage": { "input_tokens": 20974, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 4006.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 4.657 }, { "case_id": "dsh__deepseek-flash:repeat-3", "phase": "reader:model", "state_hash": "4318ff768ad4770c5e9ccf3bbc1432f8a7490ad642c4fd957f082527578f2981", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9957616172223154, "probabilities": { "yes": 0.0005167408469178446, "auth": 0.9993438395195107, "timeout": 2.897469168097517e-05, "insufficient_evidence": 0.00011044494189053605 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.9808931924534953, "probabilities": { "err": 0.9960867486628185, "ok": 9.471921901049739e-05, "missing": 0.0037238966034506427, "insufficient_evidence": 9.463551472031005e-05 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9972811165280051, "probabilities": { "1.364": 0.9995537451210661, "1.000": 0.0002410215836486913, "1.455": 8.580522274735894e-05, "0.733": 0.00010710266901123174, "insufficient_evidence": 1.2325403526719724e-05 } }, "ds4": { "type": "choice", "choice": "52430", "confidence": 0.5347325708491152, "probabilities": { "52430": 0.5341329370145809, "47158": 0.45517820861331815, "95586": 0.002696007207265465, "27616": 0.007759477849010283, "insufficient_evidence": 0.0002333693158252244 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.8992246627488175, "probabilities": { "47158": 0.9691270970353932, "52430": 0.018717963654380693, "95586": 0.0007954821407022998, "27638": 0.01121043717471444, "insufficient_evidence": 0.00014901999480933768 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9966172353526817, "probabilities": { "pearl": 0.9994998970482232, "black": 0.00036233020727864613, "edge": 5.8809060331722507e-05, "insufficient_evidence": 7.896368416643475e-05 } } }, "usage": { "input_tokens": 8563, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1310.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.311 } ], "metrics": { "requests": 15, "completed_requests": 15, "input_tokens": 118859, "output_tokens": 0, "median_latency_ms": 1247.6, "wall_s": 32.356, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "wall_s": 32.555 }