{ "schema_version": 2, "project": "compaction-check", "subset": "zen", "repeat": 1, "timestamp": "2026-09-24T16:17:39Z", "input_hashes": { "samples/compaction-check/fixtures/questions.json": "0d348f75b58e9df36b14babdfe9ae57f192a70d2452d757385839706712f00d1", "samples/compaction-check/fixtures/sessions.jsonl": "6b6e01855f16358971ca6734810ebec8d3a7b2240d9a8c05dbfd7c3f0ca115e6" }, "source_hashes": { "samples/_shared/community.py": "88155b9fcb87dbc2af50a59a02cdcc85a87e9404debaa6e6e3a96744c104d2cd", "samples/_shared/jev.py": "6bd98ea32afdccc886148ee92aaa1213a4ed2c9d2c797739450a1a1856d16a06", "samples/compaction-check/build_excerpts.py": "28a991f6ddcd8324b2336543bfde6f3d3ba379dd78c45a876d24d93d54d1e981", "samples/compaction-check/compact.py": "80df701cac548bf404ad02794635baab5bb7cf92b2b9e7acd92c263b6c9fd5ec" }, "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "mode": "live", "provider": "zen", "rows": [ { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "full", "context_chars": 16157, "tokens_estimate": 4040, "tokens_full_estimate": 4040, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "1": "keep", "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "16": "keep", "18": "keep", "21": "keep", "23": "keep", "25": "keep", "27": "keep", "30": "keep", "31": "keep", "33": "keep", "35": "keep", "37": "keep", "41": "keep", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7654, "output_tokens": 364, "median_latency_ms": 408.8, "wall_s": 15.41, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "last6", "context_chars": 12139, "tokens_estimate": 3035, "tokens_full_estimate": 4040, "tokens_saved_estimate": 1005, "saved_fraction": 0.24876237623762376, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "1": "drop", "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "drop", "14": "drop", "16": "drop", "18": "drop", "21": "drop", "23": "drop", "25": "drop", "27": "drop", "30": "drop", "31": "drop", "33": "drop", "35": "drop", "37": "drop", "41": "drop", "43": "drop", "45": "drop", "47": "drop", "49": "drop", "51": "drop", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5413, "output_tokens": 365, "median_latency_ms": 566.6, "wall_s": 15.568, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "trunc120", "context_chars": 12682, "tokens_estimate": 3171, "tokens_full_estimate": 4040, "tokens_saved_estimate": 869, "saved_fraction": 0.2150990099009901, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "1": "truncate", "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "truncate", "14": "truncate", "16": "truncate", "18": "truncate", "21": "truncate", "23": "truncate", "25": "truncate", "27": "truncate", "30": "truncate", "31": "truncate", "33": "truncate", "35": "truncate", "37": "truncate", "41": "truncate", "43": "truncate", "45": "truncate", "47": "truncate", "49": "truncate", "51": "truncate", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5858, "output_tokens": 365, "median_latency_ms": 511.9, "wall_s": 15.513, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "model", "context_chars": 16387, "tokens_estimate": 4097, "tokens_full_estimate": 4040, "tokens_saved_estimate": -57, "saved_fraction": -0.01410891089108911, "decisions": { "keep": 21, "truncate": 0, "drop": 9 }, "result_decisions": { "1": "keep", "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "drop", "14": "drop", "16": "drop", "18": "keep", "21": "drop", "23": "drop", "25": "keep", "27": "keep", "30": "drop", "31": "drop", "33": "drop", "35": "keep", "37": "keep", "41": "keep", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "drop", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 17620, "output_tokens": 1292, "median_latency_ms": 598.5, "wall_s": 31.199, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 9917, "output_tokens": 928, "median_latency_ms": 578.5, "wall_s": 15.58, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 7787, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "full", "context_chars": 9094, "tokens_estimate": 2274, "tokens_full_estimate": 2274, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 10, "truncate": 0, "drop": 0 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4316, "output_tokens": 278, "median_latency_ms": 522.1, "wall_s": 15.523, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "last6", "context_chars": 8362, "tokens_estimate": 2091, "tokens_full_estimate": 2274, "tokens_saved_estimate": 183, "saved_fraction": 0.08047493403693931, "decisions": { "keep": 6, "truncate": 0, "drop": 4 }, "result_decisions": { "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4011, "output_tokens": 278, "median_latency_ms": 410.3, "wall_s": 15.411, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "trunc120", "context_chars": 8718, "tokens_estimate": 2180, "tokens_full_estimate": 2274, "tokens_saved_estimate": 94, "saved_fraction": 0.04133685136323659, "decisions": { "keep": 6, "truncate": 4, "drop": 0 }, "result_decisions": { "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4155, "output_tokens": 278, "median_latency_ms": 529.5, "wall_s": 15.531, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "model", "context_chars": 8772, "tokens_estimate": 2193, "tokens_full_estimate": 2274, "tokens_saved_estimate": 81, "saved_fraction": 0.03562005277044855, "decisions": { "keep": 4, "truncate": 0, "drop": 6 }, "result_decisions": { "3": "drop", "5": "keep", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "drop", "19": "drop", "22": "keep", "24": "drop" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 8859, "output_tokens": 587, "median_latency_ms": 527.3, "wall_s": 31.057, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4671, "output_tokens": 309, "median_latency_ms": 501.5, "wall_s": 15.503, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 3674, "selector_reuse_break_even_estimated_reads": 46 }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "full", "context_chars": 19813, "tokens_estimate": 4954, "tokens_full_estimate": 4954, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "4": "keep", "6": "keep", "8": "keep", "10": "keep", "13": "keep", "15": "keep", "17": "keep", "19": "keep", "22": "keep", "23": "keep", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "keep", "36": "keep", "39": "keep", "41": "keep", "45": "keep", "46": "keep", "48": "keep", "51": "keep", "52": "keep", "54": "keep", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "52430", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 9021, "output_tokens": 380, "median_latency_ms": 530.0, "wall_s": 15.532, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "last6", "context_chars": 15312, "tokens_estimate": 3828, "tokens_full_estimate": 4954, "tokens_saved_estimate": 1126, "saved_fraction": 0.2272910779168349, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "4": "drop", "6": "drop", "8": "drop", "10": "drop", "13": "drop", "15": "drop", "17": "drop", "19": "drop", "22": "drop", "23": "drop", "26": "drop", "28": "drop", "30": "drop", "32": "drop", "34": "drop", "36": "drop", "39": "drop", "41": "drop", "45": "drop", "46": "drop", "48": "drop", "51": "drop", "52": "drop", "54": "drop", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 2, "correct": 2, "abstained": 4, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 4, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 1, "correct": 1, "abstained": 4, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 4 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": false, "gold": "err", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds3", "kind": "tool_dependent", "retained": false, "gold": "1.364", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds4", "kind": "tool_dependent", "retained": false, "gold": "52430", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 6444, "output_tokens": 396, "median_latency_ms": 602.1, "wall_s": 15.604, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "trunc120", "context_chars": 17221, "tokens_estimate": 4306, "tokens_full_estimate": 4954, "tokens_saved_estimate": 648, "saved_fraction": 0.1308033911990311, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "4": "truncate", "6": "truncate", "8": "truncate", "10": "truncate", "13": "truncate", "15": "truncate", "17": "truncate", "19": "truncate", "22": "truncate", "23": "truncate", "26": "truncate", "28": "truncate", "30": "truncate", "32": "truncate", "34": "truncate", "36": "truncate", "39": "truncate", "41": "truncate", "45": "truncate", "46": "truncate", "48": "truncate", "51": "truncate", "52": "truncate", "54": "truncate", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "52430", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7429, "output_tokens": 380, "median_latency_ms": 456.6, "wall_s": 15.458, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "model", "context_chars": 19813, "tokens_estimate": 4954, "tokens_full_estimate": 4954, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "4": "keep", "6": "keep", "8": "keep", "10": "keep", "13": "keep", "15": "keep", "17": "keep", "19": "keep", "22": "keep", "23": "keep", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "keep", "36": "keep", "39": "keep", "41": "keep", "45": "keep", "46": "keep", "48": "keep", "51": "keep", "52": "keep", "54": "keep", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "52430", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 20277, "output_tokens": 1310, "median_latency_ms": 624.0, "wall_s": 31.251, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 11256, "output_tokens": 930, "median_latency_ms": 659.6, "wall_s": 15.661, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 8763, "selector_reuse_break_even_estimated_reads": null } ], "summary": { "methods": { "full": { "sessions": 3, "tokens_estimate": 11268, "tokens_full_estimate": 11268, "saved_fraction": 0.0, "questions": 17, "answered": 17, "retained": 17, "correct": 17, "abstained": 0, "answerability_correct": 17, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 11, "correct": 11, "abstained": 0, "answerability_correct": 11, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "last6": { "sessions": 3, "tokens_estimate": 8954, "tokens_full_estimate": 11268, "saved_fraction": 0.20536031238906638, "questions": 17, "answered": 17, "retained": 11, "correct": 12, "abstained": 5, "answerability_correct": 16, "unsupported_correct_guess": 1, "lost_answer": 5, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 5, "correct": 6, "abstained": 5, "answerability_correct": 10, "unsupported_correct_guess": 1, "lost_answer": 5 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "trunc120": { "sessions": 3, "tokens_estimate": 9657, "tokens_full_estimate": 11268, "saved_fraction": 0.14297124600638977, "questions": 17, "answered": 17, "retained": 15, "correct": 16, "abstained": 1, "answerability_correct": 16, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 9, "correct": 10, "abstained": 1, "answerability_correct": 10, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "model": { "sessions": 3, "tokens_estimate": 11244, "tokens_full_estimate": 11268, "saved_fraction": 0.0021299254526091493, "questions": 17, "answered": 17, "retained": 17, "correct": 17, "abstained": 0, "answerability_correct": 17, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 11, "correct": 11, "abstained": 0, "answerability_correct": 11, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } } } }, "policy": { "version": "compaction-v2", "methods": [ "full", "last6", "trunc120", "model" ], "keep_last_results": 6, "truncate_chars": 120, "model_decisions": [ "keep", "drop" ], "token_estimate": "ceil(rendered context characters / 4); not provider token usage", "association": "singleton balanced visible group with no history gap; otherwise unknown", "evidence": "any annotated alternative whose event-specific spans all survive", "scope": "three committed scrubbed excerpts; no full-transcript extrapolation" }, "status": "complete", "calls": [ { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:full", "state_hash": "09f53da178caf35ec8ab71f38ae731fb32bee8d62f2b4f473e617cde73e00f1d", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 1, "probabilities": { "yes": 0, "unknown": 0, "no": 1, "insufficient_evidence": 0 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.96, "probabilities": { "none": 0, "commons": 0.97, "insufficient_evidence": 0.03, "generated": 0, "sony": 0 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 1, "probabilities": { "209394": 1, "223434": 0, "233659": 0, "379231": 0, "insufficient_evidence": 0 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 1, "probabilities": { "27638": 0, "47158": 0, "95586": 1, "95590": 0, "insufficient_evidence": 0 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.99, "probabilities": { "groups": 1, "none": 0, "gloss": 0, "all": 0, "insufficient_evidence": 0 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.91, "probabilities": { "yes": 0.94, "no": 0.03, "insufficient_evidence": 0.03 } } }, "usage": { "input_tokens": 7654, "output_tokens": 364 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 408.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.41 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:last6", "state_hash": "d92967ac2586f77e2370c343f2d044ea4cab43cbed4ae6082644954af3502493", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.89, "probabilities": { "no": 0.92, "yes": 0.01, "unknown": 0, "insufficient_evidence": 0.07 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.98, "probabilities": { "sony": 0, "generated": 0, "commons": 0.98, "none": 0, "insufficient_evidence": 0.02 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 1, "probabilities": { "209394": 0, "223434": 0, "233659": 0, "379231": 0, "insufficient_evidence": 1 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 1, "probabilities": { "27638": 0, "47158": 0, "95586": 1, "95590": 0, "insufficient_evidence": 0 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.99, "probabilities": { "gloss": 0, "none": 0, "all": 0, "groups": 1, "insufficient_evidence": 0 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.9, "probabilities": { "no": 0.03, "yes": 0.94, "insufficient_evidence": 0.03 } } }, "usage": { "input_tokens": 5413, "output_tokens": 365 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 566.6, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.568 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:trunc120", "state_hash": "308d7e00b7db233c2c64067c428d6533c53b57192d415d967f011d0d6a1a30df", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.8, "probabilities": { "no": 0.85, "yes": 0.12, "insufficient_evidence": 0.03, "unknown": 0 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.97, "probabilities": { "sony": 0, "insufficient_evidence": 0.02, "commons": 0.98, "none": 0, "generated": 0 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 1, "probabilities": { "209394": 0, "223434": 0, "233659": 0, "379231": 0, "insufficient_evidence": 1 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 1, "probabilities": { "27638": 0, "47158": 0, "95586": 1, "95590": 0, "insufficient_evidence": 0 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.99, "probabilities": { "gloss": 0, "groups": 1, "insufficient_evidence": 0, "none": 0, "all": 0 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.89, "probabilities": { "no": 0.05, "yes": 0.93, "insufficient_evidence": 0.02 } } }, "usage": { "input_tokens": 5858, "output_tokens": 365 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 511.9, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.513 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "select", "state_hash": "1ab156da003911d9ea375ce54aacbf19201cc6adba8bc73315f688d4ed39937a", "questions_hash": "b934e72498fb1f126d8c79bf9a0c69e5e573708edd8658a1f5e2ae40d4d4cb83", "question_ids": [ "result_1", "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_16", "result_18", "result_21", "result_23", "result_25", "result_27", "result_30", "result_31", "result_33", "result_35", "result_37", "result_41", "result_43", "result_45", "result_47", "result_49", "result_51", "result_53", "result_56", "result_58", "result_60", "result_62", "result_64" ], "answers": { "result_1": { "type": "choice", "choice": "keep", "confidence": 0.94, "probabilities": { "drop": 0.03, "keep": 0.97 } }, "result_3": { "type": "choice", "choice": "keep", "confidence": 0.56, "probabilities": { "drop": 0.22, "keep": 0.78 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.51, "probabilities": { "drop": 0.25, "keep": 0.75 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.85, "probabilities": { "drop": 0.08, "keep": 0.92 } }, "result_9": { "type": "choice", "choice": "keep", "confidence": 0.61, "probabilities": { "drop": 0.2, "keep": 0.8 } }, "result_12": { "type": "choice", "choice": "drop", "confidence": 0.49, "probabilities": { "drop": 0.74, "keep": 0.26 } }, "result_14": { "type": "choice", "choice": "drop", "confidence": 0.34, "probabilities": { "drop": 0.67, "keep": 0.33 } }, "result_16": { "type": "choice", "choice": "drop", "confidence": 0.2, "probabilities": { "drop": 0.6, "keep": 0.4 } }, "result_18": { "type": "choice", "choice": "keep", "confidence": 0.01, "probabilities": { "drop": 0.5, "keep": 0.5 } }, "result_21": { "type": "choice", "choice": "drop", "confidence": 0.38, "probabilities": { "drop": 0.69, "keep": 0.31 } }, "result_23": { "type": "choice", "choice": "drop", "confidence": 0.32, "probabilities": { "drop": 0.66, "keep": 0.34 } }, "result_25": { "type": "choice", "choice": "keep", "confidence": 0.43, "probabilities": { "drop": 0.28, "keep": 0.72 } }, "result_27": { "type": "choice", "choice": "keep", "confidence": 0.49, "probabilities": { "drop": 0.25, "keep": 0.75 } }, "result_30": { "type": "choice", "choice": "drop", "confidence": 0.47, "probabilities": { "drop": 0.74, "keep": 0.26 } }, "result_31": { "type": "choice", "choice": "drop", "confidence": 0.17, "probabilities": { "drop": 0.58, "keep": 0.42 } }, "result_33": { "type": "choice", "choice": "drop", "confidence": 0.21, "probabilities": { "drop": 0.6, "keep": 0.4 } }, "result_35": { "type": "choice", "choice": "keep", "confidence": 0.02, "probabilities": { "drop": 0.49, "keep": 0.51 } }, "result_37": { "type": "choice", "choice": "keep", "confidence": 0.01, "probabilities": { "drop": 0.49, "keep": 0.51 } }, "result_41": { "type": "choice", "choice": "keep", "confidence": 0.16, "probabilities": { "drop": 0.42, "keep": 0.58 } }, "result_43": { "type": "choice", "choice": "keep", "confidence": 0.44, "probabilities": { "drop": 0.28, "keep": 0.72 } }, "result_45": { "type": "choice", "choice": "keep", "confidence": 0.58, "probabilities": { "drop": 0.21, "keep": 0.79 } }, "result_47": { "type": "choice", "choice": "keep", "confidence": 0.12, "probabilities": { "drop": 0.44, "keep": 0.56 } }, "result_49": { "type": "choice", "choice": "keep", "confidence": 0.52, "probabilities": { "drop": 0.24, "keep": 0.76 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.26, "probabilities": { "drop": 0.37, "keep": 0.63 } }, "result_53": { "type": "choice", "choice": "keep", "confidence": 0.1, "probabilities": { "drop": 0.45, "keep": 0.55 } }, "result_56": { "type": "choice", "choice": "keep", "confidence": 0.94, "probabilities": { "drop": 0.03, "keep": 0.97 } }, "result_58": { "type": "choice", "choice": "keep", "confidence": 0.31, "probabilities": { "drop": 0.35, "keep": 0.65 } }, "result_60": { "type": "choice", "choice": "keep", "confidence": 0.89, "probabilities": { "drop": 0.05, "keep": 0.95 } }, "result_62": { "type": "choice", "choice": "drop", "confidence": 0.03, "probabilities": { "drop": 0.51, "keep": 0.49 } }, "result_64": { "type": "choice", "choice": "keep", "confidence": 0.79, "probabilities": { "drop": 0.1, "keep": 0.9 } } }, "usage": { "input_tokens": 9917, "output_tokens": 928 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 578.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.58 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:model", "state_hash": "b1839c3a0c7eea36a9368e884aa2be4dd2603fc74f5602495ed4fe67e1567435", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 1, "probabilities": { "insufficient_evidence": 0, "unknown": 0, "yes": 0, "no": 1 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.96, "probabilities": { "sony": 0, "insufficient_evidence": 0.03, "commons": 0.97, "generated": 0, "none": 0 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 1, "probabilities": { "209394": 1, "223434": 0, "233659": 0, "379231": 0, "insufficient_evidence": 0 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 1, "probabilities": { "27638": 0, "47158": 0, "95586": 1, "95590": 0, "insufficient_evidence": 0 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.99, "probabilities": { "insufficient_evidence": 0, "gloss": 0, "groups": 1, "all": 0, "none": 0 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.9, "probabilities": { "insufficient_evidence": 0.02, "yes": 0.93, "no": 0.04 } } }, "usage": { "input_tokens": 7703, "output_tokens": 364 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 618.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.619 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:full", "state_hash": "818641e42779c98df892775aec9b311b7dd75a0f45931036724b66d247b82d59", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.94, "probabilities": { "insufficient_evidence": 0.04, "reference": 0.96, "render": 0, "none": 0 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.83, "probabilities": { "download": 0.87, "insufficient_evidence": 0.13, "omitted": 0, "drawn": 0 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.93, "probabilities": { "patch": 0, "insufficient_evidence": 0.05, "echo": 0, "script": 0.95 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 1, "probabilities": { "27616": 1, "27638": 0, "47158": 0, "95586": 0, "insufficient_evidence": 0 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 1, "probabilities": { "notrun": 0, "insufficient_evidence": 0, "invalid": 0, "valid": 1 } } }, "usage": { "input_tokens": 4316, "output_tokens": 278 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 522.1, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.523 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:last6", "state_hash": "37c153d6143d560dec7b586528ae5881e640484c9fb38cbdb6b22ac0ed8ce18b", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.87, "probabilities": { "render": 0.01, "reference": 0.9, "insufficient_evidence": 0.09, "none": 0 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.74, "probabilities": { "download": 0.81, "omitted": 0, "insufficient_evidence": 0.19, "drawn": 0 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.93, "probabilities": { "echo": 0, "patch": 0, "insufficient_evidence": 0.05, "script": 0.95 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 1, "probabilities": { "27616": 1, "27638": 0, "47158": 0, "95586": 0, "insufficient_evidence": 0 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 1, "probabilities": { "notrun": 0, "insufficient_evidence": 0, "invalid": 0, "valid": 1 } } }, "usage": { "input_tokens": 4011, "output_tokens": 278 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 410.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.411 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:trunc120", "state_hash": "f3efcd2121b75907229d1975779876597872e4ff72d0dd63699692848b9fae3f", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.94, "probabilities": { "insufficient_evidence": 0.04, "render": 0, "none": 0, "reference": 0.96 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.81, "probabilities": { "insufficient_evidence": 0.14, "download": 0.86, "drawn": 0, "omitted": 0 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.94, "probabilities": { "echo": 0, "insufficient_evidence": 0.04, "patch": 0, "script": 0.96 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 1, "probabilities": { "27616": 1, "27638": 0, "47158": 0, "95586": 0, "insufficient_evidence": 0 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 1, "probabilities": { "insufficient_evidence": 0, "notrun": 0, "valid": 1, "invalid": 0 } } }, "usage": { "input_tokens": 4155, "output_tokens": 278 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 529.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.531 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "select", "state_hash": "f3b419d113ea7359abd8053f120ba9422237f5cf245aba011093c383b4fc09ec", "questions_hash": "9cac56873501140b02d037d69f22e218df2321487095362f67a020b3f35c89ef", "question_ids": [ "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_17", "result_19", "result_22", "result_24" ], "answers": { "result_3": { "type": "choice", "choice": "drop", "confidence": 0.41, "probabilities": { "keep": 0.3, "drop": 0.7 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.36, "probabilities": { "keep": 0.68, "drop": 0.32 } }, "result_7": { "type": "choice", "choice": "drop", "confidence": 0.34, "probabilities": { "keep": 0.33, "drop": 0.67 } }, "result_9": { "type": "choice", "choice": "drop", "confidence": 0.33, "probabilities": { "keep": 0.33, "drop": 0.67 } }, "result_12": { "type": "choice", "choice": "keep", "confidence": 0.43, "probabilities": { "keep": 0.71, "drop": 0.29 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.53, "probabilities": { "keep": 0.77, "drop": 0.23 } }, "result_17": { "type": "choice", "choice": "drop", "confidence": 0.37, "probabilities": { "keep": 0.32, "drop": 0.68 } }, "result_19": { "type": "choice", "choice": "drop", "confidence": 0.32, "probabilities": { "keep": 0.34, "drop": 0.66 } }, "result_22": { "type": "choice", "choice": "keep", "confidence": 0.96, "probabilities": { "keep": 0.98, "drop": 0.02 } }, "result_24": { "type": "choice", "choice": "drop", "confidence": 0.1, "probabilities": { "keep": 0.45, "drop": 0.55 } } }, "usage": { "input_tokens": 4671, "output_tokens": 309 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 501.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.503 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:model", "state_hash": "922f0f1b6c1f7b15d228cf704ddb39c80df8dcb9f2321db95b3419de555fa093", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.89, "probabilities": { "insufficient_evidence": 0.07, "reference": 0.93, "none": 0, "render": 0 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.8, "probabilities": { "insufficient_evidence": 0.15, "drawn": 0, "omitted": 0, "download": 0.85 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.96, "probabilities": { "insufficient_evidence": 0.03, "echo": 0, "script": 0.97, "patch": 0 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 1, "probabilities": { "27616": 1, "27638": 0, "47158": 0, "95586": 0, "insufficient_evidence": 0 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 1, "probabilities": { "insufficient_evidence": 0, "notrun": 0, "invalid": 0, "valid": 1 } } }, "usage": { "input_tokens": 4188, "output_tokens": 278 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 553.1, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.554 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:full", "state_hash": "2c718871c891948a99e03c50664a5cacbfa33a885c063e9ae0760a8b166fb44a", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.77, "probabilities": { "auth": 0.83, "timeout": 0, "yes": 0.17, "insufficient_evidence": 0 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.94, "probabilities": { "missing": 0.04, "err": 0.96, "insufficient_evidence": 0, "ok": 0 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.99, "probabilities": { "1.364": 1, "1.455": 0, "0.733": 0, "insufficient_evidence": 0, "1.000": 0 } }, "ds4": { "type": "choice", "choice": "52430", "confidence": 0.95, "probabilities": { "27616": 0, "47158": 0.02, "52430": 0.97, "95586": 0, "insufficient_evidence": 0.01 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 1, "probabilities": { "27638": 0, "47158": 1, "52430": 0, "95586": 0, "insufficient_evidence": 0 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.99, "probabilities": { "pearl": 0.99, "edge": 0, "black": 0, "insufficient_evidence": 0.01 } } }, "usage": { "input_tokens": 9021, "output_tokens": 380 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 530.0, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.532 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:last6", "state_hash": "f746af707c79a9b701f3828e24bbb0a2f7b6e6f79d063fdc225ec7a0a07a57c9", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.68, "probabilities": { "yes": 0.24, "timeout": 0, "insufficient_evidence": 0.76, "auth": 0 } }, "ds2": { "type": "choice", "choice": "insufficient_evidence", "confidence": 1, "probabilities": { "missing": 0, "ok": 0, "err": 0, "insufficient_evidence": 1 } }, "ds3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 1, "probabilities": { "1.000": 0, "1.364": 0, "insufficient_evidence": 1, "0.733": 0, "1.455": 0 } }, "ds4": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.84, "probabilities": { "27616": 0, "47158": 0.11, "52430": 0, "95586": 0.01, "insufficient_evidence": 0.88 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 1, "probabilities": { "27638": 0, "47158": 1, "52430": 0, "95586": 0, "insufficient_evidence": 0 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.99, "probabilities": { "pearl": 0.99, "edge": 0, "black": 0, "insufficient_evidence": 0.01 } } }, "usage": { "input_tokens": 6444, "output_tokens": 396 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 602.1, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.604 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:trunc120", "state_hash": "637b10b060d11f04ea8ac550b6678f62ddfc5a4b1ef52ebda9f99a98a5dea062", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.79, "probabilities": { "insufficient_evidence": 0, "auth": 0.84, "timeout": 0, "yes": 0.16 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.96, "probabilities": { "missing": 0.02, "err": 0.98, "insufficient_evidence": 0, "ok": 0 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.99, "probabilities": { "1.000": 0, "insufficient_evidence": 0.01, "1.364": 0.99, "1.455": 0, "0.733": 0 } }, "ds4": { "type": "choice", "choice": "52430", "confidence": 0.94, "probabilities": { "27616": 0, "47158": 0.02, "52430": 0.96, "95586": 0, "insufficient_evidence": 0.02 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 1, "probabilities": { "27638": 0, "47158": 1, "52430": 0, "95586": 0, "insufficient_evidence": 0 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.99, "probabilities": { "insufficient_evidence": 0, "edge": 0, "pearl": 1, "black": 0 } } }, "usage": { "input_tokens": 7429, "output_tokens": 380 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 456.6, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.458 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "select", "state_hash": "6d057db834e25182a5101a4d62be8256aafc4528a6356d6b34f5d0f2becfacc5", "questions_hash": "7f2e216b1bdff1cdf9c33d7a6e5296b61b3ef74b7ce862303c290a1473566dd9", "question_ids": [ "result_4", "result_6", "result_8", "result_10", "result_13", "result_15", "result_17", "result_19", "result_22", "result_23", "result_26", "result_28", "result_30", "result_32", "result_34", "result_36", "result_39", "result_41", "result_45", "result_46", "result_48", "result_51", "result_52", "result_54", "result_56", "result_59", "result_61", "result_63", "result_65", "result_67" ], "answers": { "result_4": { "type": "choice", "choice": "keep", "confidence": 0.9, "probabilities": { "drop": 0.05, "keep": 0.95 } }, "result_6": { "type": "choice", "choice": "keep", "confidence": 0.91, "probabilities": { "drop": 0.04, "keep": 0.96 } }, "result_8": { "type": "choice", "choice": "keep", "confidence": 0.76, "probabilities": { "drop": 0.12, "keep": 0.88 } }, "result_10": { "type": "choice", "choice": "keep", "confidence": 0.79, "probabilities": { "drop": 0.11, "keep": 0.89 } }, "result_13": { "type": "choice", "choice": "keep", "confidence": 0.65, "probabilities": { "drop": 0.17, "keep": 0.83 } }, "result_15": { "type": "choice", "choice": "keep", "confidence": 0.52, "probabilities": { "drop": 0.24, "keep": 0.76 } }, "result_17": { "type": "choice", "choice": "keep", "confidence": 0.91, "probabilities": { "drop": 0.05, "keep": 0.95 } }, "result_19": { "type": "choice", "choice": "keep", "confidence": 0.92, "probabilities": { "drop": 0.04, "keep": 0.96 } }, "result_22": { "type": "choice", "choice": "keep", "confidence": 0.36, "probabilities": { "drop": 0.32, "keep": 0.68 } }, "result_23": { "type": "choice", "choice": "keep", "confidence": 0.6, "probabilities": { "drop": 0.2, "keep": 0.8 } }, "result_26": { "type": "choice", "choice": "keep", "confidence": 0.89, "probabilities": { "drop": 0.05, "keep": 0.95 } }, "result_28": { "type": "choice", "choice": "keep", "confidence": 0.21, "probabilities": { "drop": 0.4, "keep": 0.6 } }, "result_30": { "type": "choice", "choice": "keep", "confidence": 0.62, "probabilities": { "drop": 0.19, "keep": 0.81 } }, "result_32": { "type": "choice", "choice": "keep", "confidence": 0.63, "probabilities": { "drop": 0.18, "keep": 0.82 } }, "result_34": { "type": "choice", "choice": "keep", "confidence": 0.55, "probabilities": { "drop": 0.23, "keep": 0.77 } }, "result_36": { "type": "choice", "choice": "keep", "confidence": 0.4, "probabilities": { "drop": 0.3, "keep": 0.7 } }, "result_39": { "type": "choice", "choice": "keep", "confidence": 0.2, "probabilities": { "drop": 0.4, "keep": 0.6 } }, "result_41": { "type": "choice", "choice": "keep", "confidence": 0.47, "probabilities": { "drop": 0.26, "keep": 0.74 } }, "result_45": { "type": "choice", "choice": "keep", "confidence": 0.68, "probabilities": { "drop": 0.16, "keep": 0.84 } }, "result_46": { "type": "choice", "choice": "keep", "confidence": 0.62, "probabilities": { "drop": 0.19, "keep": 0.81 } }, "result_48": { "type": "choice", "choice": "keep", "confidence": 0.76, "probabilities": { "drop": 0.12, "keep": 0.88 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.14, "probabilities": { "drop": 0.43, "keep": 0.57 } }, "result_52": { "type": "choice", "choice": "keep", "confidence": 0.41, "probabilities": { "drop": 0.3, "keep": 0.7 } }, "result_54": { "type": "choice", "choice": "keep", "confidence": 0.92, "probabilities": { "drop": 0.04, "keep": 0.96 } }, "result_56": { "type": "choice", "choice": "keep", "confidence": 0.4, "probabilities": { "drop": 0.3, "keep": 0.7 } }, "result_59": { "type": "choice", "choice": "keep", "confidence": 0.58, "probabilities": { "drop": 0.21, "keep": 0.79 } }, "result_61": { "type": "choice", "choice": "keep", "confidence": 0.09, "probabilities": { "drop": 0.46, "keep": 0.54 } }, "result_63": { "type": "choice", "choice": "keep", "confidence": 0.86, "probabilities": { "drop": 0.07, "keep": 0.93 } }, "result_65": { "type": "choice", "choice": "keep", "confidence": 0.4, "probabilities": { "drop": 0.3, "keep": 0.7 } }, "result_67": { "type": "choice", "choice": "keep", "confidence": 0.75, "probabilities": { "drop": 0.12, "keep": 0.88 } } }, "usage": { "input_tokens": 11256, "output_tokens": 930 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 659.6, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.661 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:model", "state_hash": "2c718871c891948a99e03c50664a5cacbfa33a885c063e9ae0760a8b166fb44a", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.74, "probabilities": { "timeout": 0, "yes": 0.19, "auth": 0.81, "insufficient_evidence": 0 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.94, "probabilities": { "missing": 0.05, "ok": 0, "insufficient_evidence": 0, "err": 0.95 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.99, "probabilities": { "1.000": 0, "1.364": 1, "0.733": 0, "insufficient_evidence": 0, "1.455": 0 } }, "ds4": { "type": "choice", "choice": "52430", "confidence": 0.95, "probabilities": { "27616": 0, "47158": 0.02, "52430": 0.97, "95586": 0, "insufficient_evidence": 0.01 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 1, "probabilities": { "27638": 0, "47158": 1, "52430": 0, "95586": 0, "insufficient_evidence": 0 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.99, "probabilities": { "edge": 0, "black": 0, "pearl": 0.99, "insufficient_evidence": 0.01 } } }, "usage": { "input_tokens": 9021, "output_tokens": 380 }, "model": "jev-1.13-free", "mode": "live", "provider": "zen", "label": "REAL JEV (OpenCode Zen free, jev-1.13-free)", "latency_ms": 588.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 15.59 } ], "metrics": { "requests": 15, "completed_requests": 15, "input_tokens": 101057, "output_tokens": 6273, "median_latency_ms": 530.0, "wall_s": 233.057, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "wall_s": 300.07 }