{ "schema_version": 2, "project": "compaction-check", "subset": "full", "repeat": 2, "timestamp": "2026-09-24T16:27:52Z", "input_hashes": { "samples/compaction-check/fixtures/questions.json": "0d348f75b58e9df36b14babdfe9ae57f192a70d2452d757385839706712f00d1", "samples/compaction-check/fixtures/sessions.jsonl": "6b6e01855f16358971ca6734810ebec8d3a7b2240d9a8c05dbfd7c3f0ca115e6" }, "source_hashes": { "samples/_shared/community.py": "88155b9fcb87dbc2af50a59a02cdcc85a87e9404debaa6e6e3a96744c104d2cd", "samples/_shared/jev.py": "5a0a37f5097a0ba9c7ccfc4a75990c7d825c809e48f2e407341526b6b8bbcc28", "samples/compaction-check/build_excerpts.py": "28a991f6ddcd8324b2336543bfde6f3d3ba379dd78c45a876d24d93d54d1e981", "samples/compaction-check/compact.py": "80df701cac548bf404ad02794635baab5bb7cf92b2b9e7acd92c263b6c9fd5ec" }, "label": "openjev (Codiv free hosted) - NOT JEV", "mode": "live", "provider": "codiv", "rows": [ { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 2, "method": "full", "context_chars": 16157, "tokens_estimate": 4040, "tokens_full_estimate": 4040, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "1": "keep", "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "16": "keep", "18": "keep", "21": "keep", "23": "keep", "25": "keep", "27": "keep", "30": "keep", "31": "keep", "33": "keep", "35": "keep", "37": "keep", "41": "keep", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7849, "output_tokens": 0, "median_latency_ms": 1444.9, "wall_s": 2.972, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 2, "method": "last6", "context_chars": 12139, "tokens_estimate": 3035, "tokens_full_estimate": 4040, "tokens_saved_estimate": 1005, "saved_fraction": 0.24876237623762376, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "1": "drop", "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "drop", "14": "drop", "16": "drop", "18": "drop", "21": "drop", "23": "drop", "25": "drop", "27": "drop", "30": "drop", "31": "drop", "33": "drop", "35": "drop", "37": "drop", "41": "drop", "43": "drop", "45": "drop", "47": "drop", "49": "drop", "51": "drop", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 4, "abstained": 1, "answerability_correct": 4, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5447, "output_tokens": 0, "median_latency_ms": 1304.7, "wall_s": 1.858, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 2, "method": "trunc120", "context_chars": 12682, "tokens_estimate": 3171, "tokens_full_estimate": 4040, "tokens_saved_estimate": 869, "saved_fraction": 0.2150990099009901, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "1": "truncate", "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "truncate", "14": "truncate", "16": "truncate", "18": "truncate", "21": "truncate", "23": "truncate", "25": "truncate", "27": "truncate", "30": "truncate", "31": "truncate", "33": "truncate", "35": "truncate", "37": "truncate", "41": "truncate", "43": "truncate", "45": "truncate", "47": "truncate", "49": "truncate", "51": "truncate", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5936, "output_tokens": 0, "median_latency_ms": 996.9, "wall_s": 1.688, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 2, "method": "model", "context_chars": 15579, "tokens_estimate": 3895, "tokens_full_estimate": 4040, "tokens_saved_estimate": 145, "saved_fraction": 0.03589108910891089, "decisions": { "keep": 20, "truncate": 0, "drop": 10 }, "result_decisions": { "1": "keep", "3": "drop", "5": "keep", "7": "keep", "9": "keep", "12": "drop", "14": "keep", "16": "drop", "18": "keep", "21": "drop", "23": "keep", "25": "keep", "27": "keep", "30": "drop", "31": "drop", "33": "keep", "35": "keep", "37": "keep", "41": "drop", "43": "keep", "45": "drop", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "drop", "60": "keep", "62": "keep", "64": "drop" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": false, "gold": "95586", "answer": "95590", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 25378, "output_tokens": 0, "median_latency_ms": 3332.0, "wall_s": 7.665, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 18054, "output_tokens": 0, "median_latency_ms": 3845.8, "wall_s": 4.846, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 8117, "selector_reuse_break_even_estimated_reads": 56 }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 2, "method": "full", "context_chars": 9094, "tokens_estimate": 2274, "tokens_full_estimate": 2274, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 10, "truncate": 0, "drop": 0 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4332, "output_tokens": 0, "median_latency_ms": 1312.0, "wall_s": 1.312, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 2, "method": "last6", "context_chars": 8362, "tokens_estimate": 2091, "tokens_full_estimate": 2274, "tokens_saved_estimate": 183, "saved_fraction": 0.08047493403693931, "decisions": { "keep": 6, "truncate": 0, "drop": 4 }, "result_decisions": { "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27638", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 3978, "output_tokens": 0, "median_latency_ms": 1203.8, "wall_s": 1.887, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 2, "method": "trunc120", "context_chars": 8718, "tokens_estimate": 2180, "tokens_full_estimate": 2274, "tokens_saved_estimate": 94, "saved_fraction": 0.04133685136323659, "decisions": { "keep": 6, "truncate": 4, "drop": 0 }, "result_decisions": { "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4144, "output_tokens": 0, "median_latency_ms": 742.4, "wall_s": 1.531, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 2, "method": "model", "context_chars": 9059, "tokens_estimate": 2265, "tokens_full_estimate": 2274, "tokens_saved_estimate": 9, "saved_fraction": 0.00395778364116095, "decisions": { "keep": 8, "truncate": 0, "drop": 2 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "drop", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27638", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 8941, "output_tokens": 0, "median_latency_ms": 1261.5, "wall_s": 3.948, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4635, "output_tokens": 0, "median_latency_ms": 1823.2, "wall_s": 3.075, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 3783, "selector_reuse_break_even_estimated_reads": 421 }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 2, "method": "full", "context_chars": 19813, "tokens_estimate": 4954, "tokens_full_estimate": 4954, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "4": "keep", "6": "keep", "8": "keep", "10": "keep", "13": "keep", "15": "keep", "17": "keep", "19": "keep", "22": "keep", "23": "keep", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "keep", "36": "keep", "39": "keep", "41": "keep", "45": "keep", "46": "keep", "48": "keep", "51": "keep", "52": "keep", "54": "keep", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 9347, "output_tokens": 0, "median_latency_ms": 1844.3, "wall_s": 3.135, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 2, "method": "last6", "context_chars": 15312, "tokens_estimate": 3828, "tokens_full_estimate": 4954, "tokens_saved_estimate": 1126, "saved_fraction": 0.2272910779168349, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "4": "drop", "6": "drop", "8": "drop", "10": "drop", "13": "drop", "15": "drop", "17": "drop", "19": "drop", "22": "drop", "23": "drop", "26": "drop", "28": "drop", "30": "drop", "32": "drop", "34": "drop", "36": "drop", "39": "drop", "41": "drop", "45": "drop", "46": "drop", "48": "drop", "51": "drop", "52": "drop", "54": "drop", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 2, "correct": 2, "abstained": 2, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 3, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 1, "correct": 1, "abstained": 2, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 3 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "yes", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": false, "gold": "err", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds3", "kind": "tool_dependent", "retained": false, "gold": "1.364", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds4", "kind": "tool_dependent", "retained": false, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 6599, "output_tokens": 0, "median_latency_ms": 1609.7, "wall_s": 1.752, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 2, "method": "trunc120", "context_chars": 17221, "tokens_estimate": 4306, "tokens_full_estimate": 4954, "tokens_saved_estimate": 648, "saved_fraction": 0.1308033911990311, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "4": "truncate", "6": "truncate", "8": "truncate", "10": "truncate", "13": "truncate", "15": "truncate", "17": "truncate", "19": "truncate", "22": "truncate", "23": "truncate", "26": "truncate", "28": "truncate", "30": "truncate", "32": "truncate", "34": "truncate", "36": "truncate", "39": "truncate", "41": "truncate", "45": "truncate", "46": "truncate", "48": "truncate", "51": "truncate", "52": "truncate", "54": "truncate", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7675, "output_tokens": 0, "median_latency_ms": 1270.8, "wall_s": 1.691, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 2, "method": "model", "context_chars": 17511, "tokens_estimate": 4378, "tokens_full_estimate": 4954, "tokens_saved_estimate": 576, "saved_fraction": 0.11626968106580542, "decisions": { "keep": 18, "truncate": 0, "drop": 12 }, "result_decisions": { "4": "keep", "6": "keep", "8": "drop", "10": "keep", "13": "drop", "15": "keep", "17": "keep", "19": "keep", "22": "drop", "23": "drop", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "drop", "36": "keep", "39": "keep", "41": "keep", "45": "drop", "46": "drop", "48": "keep", "51": "keep", "52": "drop", "54": "keep", "56": "drop", "59": "drop", "61": "keep", "63": "drop", "65": "keep", "67": "drop" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 2, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 2 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "yes", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "52430", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": false, "gold": "47158", "answer": "52430", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 29187, "output_tokens": 0, "median_latency_ms": 2394.9, "wall_s": 5.438, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 20974, "output_tokens": 0, "median_latency_ms": 3300.5, "wall_s": 3.948, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 9095, "selector_reuse_break_even_estimated_reads": 16 } ], "summary": { "methods": { "full": { "sessions": 3, "tokens_estimate": 11268, "tokens_full_estimate": 11268, "saved_fraction": 0.0, "questions": 17, "answered": 17, "retained": 17, "correct": 15, "abstained": 0, "answerability_correct": 15, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 11, "correct": 10, "abstained": 0, "answerability_correct": 10, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "last6": { "sessions": 3, "tokens_estimate": 8954, "tokens_full_estimate": 11268, "saved_fraction": 0.20536031238906638, "questions": 17, "answered": 17, "retained": 11, "correct": 10, "abstained": 3, "answerability_correct": 12, "unsupported_correct_guess": 1, "lost_answer": 5, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 5, "correct": 5, "abstained": 3, "answerability_correct": 7, "unsupported_correct_guess": 1, "lost_answer": 5 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "trunc120": { "sessions": 3, "tokens_estimate": 9657, "tokens_full_estimate": 11268, "saved_fraction": 0.14297124600638977, "questions": 17, "answered": 17, "retained": 15, "correct": 15, "abstained": 1, "answerability_correct": 15, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 9, "correct": 9, "abstained": 1, "answerability_correct": 9, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "model": { "sessions": 3, "tokens_estimate": 10538, "tokens_full_estimate": 11268, "saved_fraction": 0.06478523251686186, "questions": 17, "answered": 17, "retained": 14, "correct": 12, "abstained": 0, "answerability_correct": 12, "unsupported_correct_guess": 0, "lost_answer": 4, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 8, "correct": 7, "abstained": 0, "answerability_correct": 7, "unsupported_correct_guess": 0, "lost_answer": 4 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } } } }, "policy": { "version": "compaction-v2", "methods": [ "full", "last6", "trunc120", "model" ], "keep_last_results": 6, "truncate_chars": 120, "model_decisions": [ "keep", "drop" ], "token_estimate": "ceil(rendered context characters / 4); not provider token usage", "association": "singleton balanced visible group with no history gap; otherwise unknown", "evidence": "any annotated alternative whose event-specific spans all survive", "scope": "three committed scrubbed excerpts; no full-transcript extrapolation" }, "status": "complete", "calls": [ { "case_id": "claude-code__opus-5-5:repeat-2", "phase": "reader:full", "state_hash": "09f53da178caf35ec8ab71f38ae731fb32bee8d62f2b4f473e617cde73e00f1d", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9963723832063087, "probabilities": { "yes": 0.00018594024208773882, "no": 0.9994642664187803, "unknown": 3.43839281101033e-05, "insufficient_evidence": 0.0003154094110219542 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9845561197428971, "probabilities": { "commons": 0.996863212209541, "sony": 0.00019588503826778176, "none": 0.0010582415533327717, "generated": 0.0004107662603097708, "insufficient_evidence": 0.0014718949385486757 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.505261218848133, "probabilities": { "209394": 0.5245733542549375, "223434": 0.004579340764196158, "233659": 0.014625205943244136, "379231": 0.45413544572691933, "insufficient_evidence": 0.00208665331070288 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.7617746679458229, "probabilities": { "95586": 0.9148409845442949, "95590": 0.033766237911220215, "47158": 0.005562045255382445, "27638": 0.040028701044001025, "insufficient_evidence": 0.005802031245101383 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9860415674161122, "probabilities": { "groups": 0.9970209965225765, "none": 0.002288277239662769, "gloss": 0.00026976252501628153, "all": 0.00039803875776883794, "insufficient_evidence": 2.2924954975595153e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.8625283486857155, "probabilities": { "yes": 0.0006772217002821204, "no": 0.9662009535861531, "insufficient_evidence": 0.03312182471356479 } } }, "usage": { "input_tokens": 7849, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1444.9, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.972 }, { "case_id": "claude-code__opus-5-5:repeat-2", "phase": "reader:last6", "state_hash": "d92967ac2586f77e2370c343f2d044ea4cab43cbed4ae6082644954af3502493", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9635875274126247, "probabilities": { "yes": 0.005318424151865425, "no": 0.9923924431446316, "unknown": 0.0004485751482788455, "insufficient_evidence": 0.0018405575552240406 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9929189072246813, "probabilities": { "commons": 0.9986880651125049, "sony": 0.00016250369970134822, "none": 0.00033051752743567256, "generated": 0.000765221554282948, "insufficient_evidence": 5.3692106075157515e-05 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9018292744036616, "probabilities": { "209394": 0.00047450132018442395, "223434": 0.007479347358233284, "233659": 0.009066670571350568, "379231": 0.010528723914025203, "insufficient_evidence": 0.9724507568362065 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.8986213040136102, "probabilities": { "95586": 0.968826045644998, "95590": 0.010532269382791821, "47158": 0.00039824458637907823, "27638": 0.019712351543911797, "insufficient_evidence": 0.0005310888419193349 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9887173478185098, "probabilities": { "groups": 0.9978187423628189, "none": 0.0009962994316465065, "gloss": 0.0006028778543319711, "all": 0.0004676559997032172, "insufficient_evidence": 0.00011442435149941707 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.6996877467277398, "probabilities": { "yes": 0.07571766813713102, "no": 0.9128124777168828, "insufficient_evidence": 0.011469854145986199 } } }, "usage": { "input_tokens": 5447, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1304.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.858 }, { "case_id": "claude-code__opus-5-5:repeat-2", "phase": "reader:trunc120", "state_hash": "308d7e00b7db233c2c64067c428d6533c53b57192d415d967f011d0d6a1a30df", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9694965254971443, "probabilities": { "yes": 0.003367096959584725, "no": 0.9940473032152924, "unknown": 0.0011487407520649805, "insufficient_evidence": 0.0014368590730577872 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9935476721671457, "probabilities": { "commons": 0.9988625121072496, "sony": 0.00019379331370204562, "none": 0.0003564739529321807, "generated": 0.0003598123567287474, "insufficient_evidence": 0.00022740826938745995 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9370783936252627, "probabilities": { "209394": 0.0009062653581555295, "223434": 0.004882303138949867, "233659": 0.005223543539609363, "379231": 0.004832490075811379, "insufficient_evidence": 0.9841553978874739 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.7725660096142103, "probabilities": { "95586": 0.8979687687426388, "95590": 0.004154822900811396, "47158": 0.001310073004873339, "27638": 0.09390099411811599, "insufficient_evidence": 0.002665341233560505 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9800829530668107, "probabilities": { "groups": 0.995687428949357, "none": 0.0028015301107956123, "gloss": 0.0007677330479615791, "all": 0.0005515048483557159, "insufficient_evidence": 0.00019180304353010196 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.7421422373827655, "probabilities": { "yes": 0.9277218836623955, "no": 0.06491446379034355, "insufficient_evidence": 0.007363652547260941 } } }, "usage": { "input_tokens": 5936, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 996.9, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.688 }, { "case_id": "claude-code__opus-5-5:repeat-2", "phase": "select", "state_hash": "1ab156da003911d9ea375ce54aacbf19201cc6adba8bc73315f688d4ed39937a", "questions_hash": "b934e72498fb1f126d8c79bf9a0c69e5e573708edd8658a1f5e2ae40d4d4cb83", "question_ids": [ "result_1", "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_16", "result_18", "result_21", "result_23", "result_25", "result_27", "result_30", "result_31", "result_33", "result_35", "result_37", "result_41", "result_43", "result_45", "result_47", "result_49", "result_51", "result_53", "result_56", "result_58", "result_60", "result_62", "result_64" ], "answers": { "result_1": { "type": "choice", "choice": "keep", "confidence": 0.7068710238106823, "probabilities": { "keep": 0.9484062318779782, "drop": 0.05159376812202181 } }, "result_3": { "type": "choice", "choice": "drop", "confidence": 0.9721158516916414, "probabilities": { "keep": 0.002812423064759506, "drop": 0.9971875769352405 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.6687185809497409, "probabilities": { "keep": 0.9390305231721152, "drop": 0.06096947682788486 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.0030621296772250783, "probabilities": { "keep": 0.5325653531283024, "drop": 0.4674346468716976 } }, "result_9": { "type": "choice", "choice": "keep", "confidence": 0.5893935800352584, "probabilities": { "keep": 0.9175923422534925, "drop": 0.08240765774650749 } }, "result_12": { "type": "choice", "choice": "drop", "confidence": 0.2649352000273283, "probabilities": { "keep": 0.2066677518102381, "drop": 0.7933322481897619 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.01477573163471324, "probabilities": { "keep": 0.5714378896545506, "drop": 0.4285621103454494 } }, "result_16": { "type": "choice", "choice": "drop", "confidence": 0.0211142077592551, "probabilities": { "keep": 0.4146661961860799, "drop": 0.5853338038139201 } }, "result_18": { "type": "choice", "choice": "keep", "confidence": 0.4581027811171401, "probabilities": { "keep": 0.8755930499110685, "drop": 0.12440695008893154 } }, "result_21": { "type": "choice", "choice": "drop", "confidence": 0.41691896069077805, "probabilities": { "keep": 0.13955836480358774, "drop": 0.8604416351964123 } }, "result_23": { "type": "choice", "choice": "keep", "confidence": 0.5267282118664983, "probabilities": { "keep": 0.8986463965499323, "drop": 0.10135360345006779 } }, "result_25": { "type": "choice", "choice": "keep", "confidence": 0.02008580897399781, "probabilities": { "keep": 0.5832396829605491, "drop": 0.41676031703945093 } }, "result_27": { "type": "choice", "choice": "keep", "confidence": 0.30754552123592616, "probabilities": { "keep": 0.814269304186418, "drop": 0.1857306958135821 } }, "result_30": { "type": "choice", "choice": "drop", "confidence": 0.0658042164323075, "probabilities": { "keep": 0.35014309179911074, "drop": 0.6498569082008893 } }, "result_31": { "type": "choice", "choice": "drop", "confidence": 0.08922418825350986, "probabilities": { "keep": 0.3259890524075713, "drop": 0.6740109475924287 } }, "result_33": { "type": "choice", "choice": "keep", "confidence": 0.05102796163259837, "probabilities": { "keep": 0.6321946839219899, "drop": 0.36780531607801015 } }, "result_35": { "type": "choice", "choice": "keep", "confidence": 0.44236760278820475, "probabilities": { "keep": 0.8699291726868185, "drop": 0.1300708273131815 } }, "result_37": { "type": "choice", "choice": "keep", "confidence": 0.20312922832688285, "probabilities": { "keep": 0.7588963663350217, "drop": 0.24110363366497828 } }, "result_41": { "type": "choice", "choice": "drop", "confidence": 0.022954637590188454, "probabilities": { "keep": 0.4110438997032726, "drop": 0.5889561002967274 } }, "result_43": { "type": "choice", "choice": "keep", "confidence": 0.028689212731232372, "probabilities": { "keep": 0.5993822860935103, "drop": 0.40061771390648976 } }, "result_45": { "type": "choice", "choice": "drop", "confidence": 0.004371458014895024, "probabilities": { "keep": 0.46109625898186124, "drop": 0.5389037410181389 } }, "result_47": { "type": "choice", "choice": "keep", "confidence": 0.037812745702474726, "probabilities": { "keep": 0.6139736825101403, "drop": 0.38602631748985966 } }, "result_49": { "type": "choice", "choice": "keep", "confidence": 0.020328925975464318, "probabilities": { "keep": 0.5837395589906187, "drop": 0.41626044100938137 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.046700989851185204, "probabilities": { "keep": 0.6265303038334135, "drop": 0.37346969616658654 } }, "result_53": { "type": "choice", "choice": "keep", "confidence": 0.001292035237062894, "probabilities": { "keep": 0.5211577787223072, "drop": 0.4788422212776929 } }, "result_56": { "type": "choice", "choice": "keep", "confidence": 0.03034097174832273, "probabilities": { "keep": 0.6021834365492924, "drop": 0.39781656345070765 } }, "result_58": { "type": "choice", "choice": "drop", "confidence": 0.015738348496685983, "probabilities": { "keep": 0.4262800396480182, "drop": 0.5737199603519818 } }, "result_60": { "type": "choice", "choice": "keep", "confidence": 0.023473743927720725, "probabilities": { "keep": 0.5899508761182022, "drop": 0.4100491238817978 } }, "result_62": { "type": "choice", "choice": "keep", "confidence": 0.13293857453505886, "probabilities": { "keep": 0.7112804730163075, "drop": 0.2887195269836925 } }, "result_64": { "type": "choice", "choice": "drop", "confidence": 0.02709138545262868, "probabilities": { "keep": 0.40340683752709183, "drop": 0.5965931624729082 } } }, "usage": { "input_tokens": 18054, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 3845.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 4.846 }, { "case_id": "claude-code__opus-5-5:repeat-2", "phase": "reader:model", "state_hash": "53796de21d206344386bd1eb3a710c79edf4a3404345eb3d1ac8f32a863b417e", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9803160806834447, "probabilities": { "yes": 0.0006097803467444075, "no": 0.9961394852933094, "unknown": 5.969287810161578e-05, "insufficient_evidence": 0.0031910414818445676 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.953139114328166, "probabilities": { "commons": 0.9882780548526402, "sony": 0.0004455767667635963, "none": 0.003133303506346221, "generated": 0.0010292230418618995, "insufficient_evidence": 0.0071138418323880735 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.7605087451856906, "probabilities": { "209394": 0.9118009191781518, "223434": 0.002580365774127659, "233659": 0.037604576881709435, "379231": 0.04268934596595155, "insufficient_evidence": 0.005324792200059669 } }, "cc4": { "type": "choice", "choice": "95590", "confidence": 0.5672640567190456, "probabilities": { "95586": 0.38552161136448354, "95590": 0.610072008135015, "47158": 0.0009395577707887195, "27638": 0.003022393041096882, "insufficient_evidence": 0.00044442968861578346 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.968587083209753, "probabilities": { "groups": 0.9929342031136504, "none": 0.002718061747142068, "gloss": 0.002365285539928611, "all": 0.0016962729523268454, "insufficient_evidence": 0.0002861766469519952 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.5422248631819462, "probabilities": { "yes": 0.07194567233190066, "no": 0.8596181771250393, "insufficient_evidence": 0.06843615054305999 } } }, "usage": { "input_tokens": 7324, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 2818.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.819 }, { "case_id": "codex__gpt-6-astra:repeat-2", "phase": "reader:full", "state_hash": "818641e42779c98df892775aec9b311b7dd75a0f45931036724b66d247b82d59", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9971060385603784, "probabilities": { "reference": 0.9995491821963666, "render": 0.00043215845494788837, "none": 4.383089758128575e-06, "insufficient_evidence": 1.4276258927467936e-05 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9989001357179531, "probabilities": { "download": 0.9998572546529031, "drawn": 7.498248314813366e-05, "omitted": 5.593426147996766e-06, "insufficient_evidence": 6.216943780083688e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9844957876533733, "probabilities": { "script": 0.9971150164844258, "patch": 0.0005338230101493693, "echo": 0.002268244593467869, "insufficient_evidence": 8.291591195690013e-05 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.9447729631470033, "probabilities": { "27616": 0.9826496508748293, "27638": 0.017136273409955768, "47158": 4.1719758553624636e-05, "95586": 0.00013839555912902794, "insufficient_evidence": 3.396039753233695e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9881531186046515, "probabilities": { "valid": 0.9978914232573705, "invalid": 0.0016976290230460219, "notrun": 0.00021537652454548092, "insufficient_evidence": 0.0001955711950380209 } } }, "usage": { "input_tokens": 4332, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1312.0, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.312 }, { "case_id": "codex__gpt-6-astra:repeat-2", "phase": "reader:last6", "state_hash": "37c153d6143d560dec7b586528ae5881e640484c9fb38cbdb6b22ac0ed8ce18b", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9986242193727686, "probabilities": { "reference": 0.9998024298586111, "render": 0.00019298139942888802, "none": 1.292473206557367e-06, "insufficient_evidence": 3.296268753519698e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9992563918929057, "probabilities": { "download": 0.9999073046219, "drawn": 2.4432237220307826e-05, "omitted": 7.635796028147043e-06, "insufficient_evidence": 6.06273448514622e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9326982301214436, "probabilities": { "script": 0.9828310110194991, "patch": 0.0014933031444901146, "echo": 0.015391771215125038, "insufficient_evidence": 0.00028391462088567693 } }, "cx4": { "type": "choice", "choice": "27638", "confidence": 0.659150726805196, "probabilities": { "27616": 0.23318099553638263, "27638": 0.7662269395032705, "47158": 0.00018199869742403587, "95586": 0.00020958517137554988, "insufficient_evidence": 0.00020048109154732375 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9971116962611678, "probabilities": { "valid": 0.9995839395465278, "invalid": 0.0002856015229430791, "notrun": 6.42477487332507e-05, "insufficient_evidence": 6.621118179587231e-05 } } }, "usage": { "input_tokens": 3978, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1203.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.887 }, { "case_id": "codex__gpt-6-astra:repeat-2", "phase": "reader:trunc120", "state_hash": "f3efcd2121b75907229d1975779876597872e4ff72d0dd63699692848b9fae3f", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.999337103474318, "probabilities": { "reference": 0.9999130445341308, "render": 8.267969678881138e-05, "none": 7.117796140788137e-07, "insufficient_evidence": 3.5639894663417285e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9989741321603014, "probabilities": { "download": 0.999867045973058, "drawn": 4.220937290626162e-05, "omitted": 5.029986906280855e-06, "insufficient_evidence": 8.571466712952477e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9703511197721205, "probabilities": { "script": 0.9936299673716457, "patch": 0.0005389306998270252, "echo": 0.005680846142025944, "insufficient_evidence": 0.0001502557865014002 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.978950954176378, "probabilities": { "27616": 0.9950191673096344, "27638": 0.004196562702312296, "47158": 2.601236958388664e-05, "95586": 0.0006954509673028648, "insufficient_evidence": 6.28066511665876e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9985154581131596, "probabilities": { "valid": 0.9998013065633506, "invalid": 0.0001213759806573165, "notrun": 1.1739288420603173e-05, "insufficient_evidence": 6.557816757129452e-05 } } }, "usage": { "input_tokens": 4144, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 742.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.531 }, { "case_id": "codex__gpt-6-astra:repeat-2", "phase": "select", "state_hash": "f3b419d113ea7359abd8053f120ba9422237f5cf245aba011093c383b4fc09ec", "questions_hash": "9cac56873501140b02d037d69f22e218df2321487095362f67a020b3f35c89ef", "question_ids": [ "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_17", "result_19", "result_22", "result_24" ], "answers": { "result_3": { "type": "choice", "choice": "keep", "confidence": 0.367394516369385, "probabilities": { "keep": 0.8407111736878708, "drop": 0.15928882631212918 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.398548174703538, "probabilities": { "keep": 0.8533268626912439, "drop": 0.14667313730875606 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.07829316208665171, "probabilities": { "keep": 0.6632171023996803, "drop": 0.33678289760031965 } }, "result_9": { "type": "choice", "choice": "drop", "confidence": 0.1415019262998275, "probabilities": { "keep": 0.282249677056535, "drop": 0.717750322943465 } }, "result_12": { "type": "choice", "choice": "keep", "confidence": 0.12893191570159213, "probabilities": { "keep": 0.7081742034153908, "drop": 0.2918257965846092 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.3093284738202411, "probabilities": { "keep": 0.8151039107230094, "drop": 0.18489608927699053 } }, "result_17": { "type": "choice", "choice": "keep", "confidence": 0.043034798949896036, "probabilities": { "keep": 0.6215147106038389, "drop": 0.3784852893961611 } }, "result_19": { "type": "choice", "choice": "drop", "confidence": 0.657809617268694, "probabilities": { "keep": 0.06375924088471246, "drop": 0.9362407591152876 } }, "result_22": { "type": "choice", "choice": "keep", "confidence": 0.7112166123287171, "probabilities": { "keep": 0.9494370841613777, "drop": 0.05056291583862224 } }, "result_24": { "type": "choice", "choice": "keep", "confidence": 5.0073966429975414e-05, "probabilities": { "keep": 0.5041658268738345, "drop": 0.49583417312616557 } } }, "usage": { "input_tokens": 4635, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1823.2, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 3.075 }, { "case_id": "codex__gpt-6-astra:repeat-2", "phase": "reader:model", "state_hash": "53dd13871353734e92447cc4b22e707964be6c8b5e8f82f805e155e95227b006", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.996426157641875, "probabilities": { "reference": 0.9994213860680279, "render": 0.0005670808708719273, "none": 2.3880732612619567e-06, "insufficient_evidence": 9.144987838999162e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9990600488301671, "probabilities": { "download": 0.9998776666402676, "drawn": 8.931543702247926e-05, "omitted": 1.9150582499278583e-06, "insufficient_evidence": 3.110286446008581e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9925274362223586, "probabilities": { "script": 0.9987616278556568, "patch": 0.00018467098020628146, "echo": 0.0009624713058740015, "insufficient_evidence": 9.122985826285298e-05 } }, "cx4": { "type": "choice", "choice": "27638", "confidence": 0.9478706567680577, "probabilities": { "27616": 0.015186402317342619, "27638": 0.9842655237840452, "47158": 0.00012186114103600594, "95586": 0.00023967442484531322, "insufficient_evidence": 0.00018653833273085378 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9910202889092798, "probabilities": { "valid": 0.9984520971398329, "invalid": 0.0012854686791993549, "notrun": 0.00012046247603412531, "insufficient_evidence": 0.0001419717049336983 } } }, "usage": { "input_tokens": 4306, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 699.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 0.873 }, { "case_id": "dsh__deepseek-flash:repeat-2", "phase": "reader:full", "state_hash": "2c718871c891948a99e03c50664a5cacbfa33a885c063e9ae0760a8b166fb44a", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9983276576744359, "probabilities": { "yes": 8.52926507454179e-05, "auth": 0.9997748636301802, "timeout": 1.7637614225581774e-05, "insufficient_evidence": 0.00012220610484876119 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.9875410674177114, "probabilities": { "err": 0.9978976812940561, "ok": 0.0006851172751697051, "missing": 0.0009689373515211671, "insufficient_evidence": 0.0004482640792530871 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9965861536988855, "probabilities": { "1.364": 0.9994226936833641, "1.000": 0.00030403404744673264, "1.455": 6.918280354150062e-05, "0.733": 0.0001872112399929529, "insufficient_evidence": 1.687822565469826e-05 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.9229442397273984, "probabilities": { "52430": 0.013959912128692228, "47158": 0.9785795208303925, "95586": 0.0010787724728563567, "27616": 0.004802602888324713, "insufficient_evidence": 0.0015791916797343042 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.9161583382232197, "probabilities": { "47158": 0.972202508467669, "52430": 0.026027348506279132, "95586": 0.0005253172807071682, "27638": 0.00117336228925132, "insufficient_evidence": 7.146345609340689e-05 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9821694577343373, "probabilities": { "pearl": 0.9966960808030642, "black": 0.00237667095408789, "edge": 0.00030135791773404277, "insufficient_evidence": 0.0006258903251139874 } } }, "usage": { "input_tokens": 9347, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1844.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 3.135 }, { "case_id": "dsh__deepseek-flash:repeat-2", "phase": "reader:last6", "state_hash": "f746af707c79a9b701f3828e24bbb0a2f7b6e6f79d063fdc225ec7a0a07a57c9", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "yes", "confidence": 0.991822471649594, "probabilities": { "yes": 0.9985968914738279, "auth": 0.00112434478899941, "timeout": 3.2481733816931016e-06, "insufficient_evidence": 0.0002755155637909684 } }, "ds2": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9056602520301783, "probabilities": { "err": 0.007743311963645706, "ok": 0.015700098569546593, "missing": 0.000562327153081673, "insufficient_evidence": 0.9759942623137261 } }, "ds3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.7270617842067119, "probabilities": { "1.364": 0.013799257863352623, "1.000": 0.03300559669357149, "1.455": 0.012619164476207484, "0.733": 0.03662800578388126, "insufficient_evidence": 0.9039479751829871 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.5842562891458254, "probabilities": { "52430": 0.21910866937945556, "47158": 0.7509327912048631, "95586": 0.001476267947581033, "27616": 0.025093341357540166, "insufficient_evidence": 0.003388930110560086 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.6652173520931581, "probabilities": { "47158": 0.8027715894486647, "52430": 0.18814794927308426, "95586": 0.0009108560852326321, "27638": 0.0074229998093792385, "insufficient_evidence": 0.0007466053836392368 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9581496526151004, "probabilities": { "pearl": 0.9914224935933817, "black": 0.00461748529067285, "edge": 0.002087778375356961, "insufficient_evidence": 0.0018722427405884255 } } }, "usage": { "input_tokens": 6599, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1609.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.752 }, { "case_id": "dsh__deepseek-flash:repeat-2", "phase": "reader:trunc120", "state_hash": "637b10b060d11f04ea8ac550b6678f62ddfc5a4b1ef52ebda9f99a98a5dea062", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9975661248289166, "probabilities": { "yes": 0.0001646426025455816, "auth": 0.9996555938884575, "timeout": 1.2408548241262408e-05, "insufficient_evidence": 0.0001673549607556823 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.9903664208565975, "probabilities": { "err": 0.9984327023573534, "ok": 0.00034724885516407663, "missing": 0.0006794029845115169, "insufficient_evidence": 0.0005406458029710182 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9942368633828877, "probabilities": { "1.364": 0.9989587155164328, "1.000": 0.0006167695356324742, "1.455": 0.00016517547853766402, "0.733": 0.00022768226168526237, "insufficient_evidence": 3.1657207711832365e-05 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.8211942554190444, "probabilities": { "52430": 0.07876083247291775, "47158": 0.9198022278361314, "95586": 0.00035814626829226514, "27616": 0.0008806891128722523, "insufficient_evidence": 0.00019810430978636784 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.9100887137987115, "probabilities": { "47158": 0.9704896172608909, "52430": 0.026655028647422807, "95586": 0.0014844328371377696, "27638": 0.0012882504955856384, "insufficient_evidence": 8.267075896283699e-05 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9913102099445629, "probabilities": { "pearl": 0.9985754647772282, "black": 0.0008787304497597872, "edge": 0.00017015026640629144, "insufficient_evidence": 0.00037565450660561944 } } }, "usage": { "input_tokens": 7675, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1270.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.691 }, { "case_id": "dsh__deepseek-flash:repeat-2", "phase": "select", "state_hash": "6d057db834e25182a5101a4d62be8256aafc4528a6356d6b34f5d0f2becfacc5", "questions_hash": "7f2e216b1bdff1cdf9c33d7a6e5296b61b3ef74b7ce862303c290a1473566dd9", "question_ids": [ "result_4", "result_6", "result_8", "result_10", "result_13", "result_15", "result_17", "result_19", "result_22", "result_23", "result_26", "result_28", "result_30", "result_32", "result_34", "result_36", "result_39", "result_41", "result_45", "result_46", "result_48", "result_51", "result_52", "result_54", "result_56", "result_59", "result_61", "result_63", "result_65", "result_67" ], "answers": { "result_4": { "type": "choice", "choice": "keep", "confidence": 0.37379860128232534, "probabilities": { "keep": 0.8433637176078849, "drop": 0.1566362823921151 } }, "result_6": { "type": "choice", "choice": "keep", "confidence": 0.0025239520975719465, "probabilities": { "keep": 0.529567294656879, "drop": 0.47043270534312104 } }, "result_8": { "type": "choice", "choice": "drop", "confidence": 0.07799923785741358, "probabilities": { "keep": 0.3370838365403296, "drop": 0.6629161634596704 } }, "result_10": { "type": "choice", "choice": "keep", "confidence": 0.46052545520913724, "probabilities": { "keep": 0.8764518825442703, "drop": 0.12354811745572966 } }, "result_13": { "type": "choice", "choice": "drop", "confidence": 0.000360284006707845, "probabilities": { "keep": 0.4888261677472202, "drop": 0.5111738322527798 } }, "result_15": { "type": "choice", "choice": "keep", "confidence": 0.12977045360312034, "probabilities": { "keep": 0.7088286555703174, "drop": 0.29117134442968257 } }, "result_17": { "type": "choice", "choice": "keep", "confidence": 0.0017132304675453458, "probabilities": { "keep": 0.5243623759631817, "drop": 0.4756376240368183 } }, "result_19": { "type": "choice", "choice": "keep", "confidence": 0.36205381677025505, "probabilities": { "keep": 0.8384746655743925, "drop": 0.1615253344256075 } }, "result_22": { "type": "choice", "choice": "drop", "confidence": 0.04730395793816655, "probabilities": { "keep": 0.37266453204185507, "drop": 0.627335467958145 } }, "result_23": { "type": "choice", "choice": "drop", "confidence": 0.0020977643626104436, "probabilities": { "keep": 0.473043047708392, "drop": 0.5269569522916079 } }, "result_26": { "type": "choice", "choice": "keep", "confidence": 0.5621597115182069, "probabilities": { "keep": 0.9095889721497843, "drop": 0.09041102785021565 } }, "result_28": { "type": "choice", "choice": "keep", "confidence": 0.012055920973662704, "probabilities": { "keep": 0.564549300468325, "drop": 0.4354506995316751 } }, "result_30": { "type": "choice", "choice": "keep", "confidence": 0.12786307673733543, "probabilities": { "keep": 0.7073366045046262, "drop": 0.2926633954953738 } }, "result_32": { "type": "choice", "choice": "keep", "confidence": 0.10402402570588665, "probabilities": { "keep": 0.6875547235099588, "drop": 0.3124452764900412 } }, "result_34": { "type": "choice", "choice": "drop", "confidence": 0.01670900276518361, "probabilities": { "keep": 0.4240493056435545, "drop": 0.5759506943564455 } }, "result_36": { "type": "choice", "choice": "keep", "confidence": 0.027852415849600076, "probabilities": { "keep": 0.5979317663248078, "drop": 0.4020682336751922 } }, "result_39": { "type": "choice", "choice": "keep", "confidence": 0.4000581488621745, "probabilities": { "keep": 0.8539204249942879, "drop": 0.14607957500571214 } }, "result_41": { "type": "choice", "choice": "keep", "confidence": 0.7513466547347308, "probabilities": { "keep": 0.958604715564262, "drop": 0.04139528443573808 } }, "result_45": { "type": "choice", "choice": "drop", "confidence": 0.7438210758099126, "probabilities": { "keep": 0.04306637140304262, "drop": 0.9569336285969574 } }, "result_46": { "type": "choice", "choice": "drop", "confidence": 0.3202403196353436, "probabilities": { "keep": 0.17985497411030893, "drop": 0.8201450258896911 } }, "result_48": { "type": "choice", "choice": "keep", "confidence": 0.017850591739482446, "probabilities": { "keep": 0.578491946798889, "drop": 0.42150805320111095 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.017151296668826643, "probabilities": { "keep": 0.5769453916165828, "drop": 0.42305460838341724 } }, "result_52": { "type": "choice", "choice": "drop", "confidence": 0.007863721863900164, "probabilities": { "keep": 0.4478425174088858, "drop": 0.5521574825911142 } }, "result_54": { "type": "choice", "choice": "keep", "confidence": 0.22550372113124073, "probabilities": { "keep": 0.7720069617093444, "drop": 0.2279930382906557 } }, "result_56": { "type": "choice", "choice": "drop", "confidence": 0.05474799721469914, "probabilities": { "keep": 0.36313161971835545, "drop": 0.6368683802816445 } }, "result_59": { "type": "choice", "choice": "drop", "confidence": 0.0039195593092705705, "probabilities": { "keep": 0.46316000678306807, "drop": 0.5368399932169319 } }, "result_61": { "type": "choice", "choice": "keep", "confidence": 0.015943759493323495, "probabilities": { "keep": 0.5741977109267172, "drop": 0.42580228907328277 } }, "result_63": { "type": "choice", "choice": "drop", "confidence": 0.02239382500323539, "probabilities": { "keep": 0.4121315306254269, "drop": 0.587868469374573 } }, "result_65": { "type": "choice", "choice": "keep", "confidence": 0.02032929100144243, "probabilities": { "keep": 0.5837403072390766, "drop": 0.4162596927609235 } }, "result_67": { "type": "choice", "choice": "drop", "confidence": 0.06872320989186931, "probabilities": { "keep": 0.3469085569446689, "drop": 0.6530914430553311 } } }, "usage": { "input_tokens": 20974, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 3300.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 3.948 }, { "case_id": "dsh__deepseek-flash:repeat-2", "phase": "reader:model", "state_hash": "75725dfb19ea1b51220753e593b99263f19e4e1f724b14338b2a5fa102fc54f8", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "yes", "confidence": 0.9553812645247857, "probabilities": { "yes": 0.9901995570038277, "auth": 0.005564870442154106, "timeout": 9.832307423870853e-06, "insufficient_evidence": 0.004225740246594307 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.8918226547080789, "probabilities": { "err": 0.9720936771203105, "ok": 0.013062809894575698, "missing": 0.013980699429488593, "insufficient_evidence": 0.000862813555625259 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9913945554244578, "probabilities": { "1.364": 0.9983455811518653, "1.000": 0.00097854376575363, "1.455": 7.536965723594304e-05, "0.733": 0.0005147455448607187, "insufficient_evidence": 8.575988028439616e-05 } }, "ds4": { "type": "choice", "choice": "52430", "confidence": 0.536375295687508, "probabilities": { "52430": 0.7759008893987156, "47158": 0.14871893611732676, "95586": 0.026219417099300757, "27616": 0.0408636188069264, "insufficient_evidence": 0.00829713857773053 } }, "ds5": { "type": "choice", "choice": "52430", "confidence": 0.5194304703283655, "probabilities": { "47158": 0.09387313404408519, "52430": 0.7758645770420736, "95586": 0.01405743418905291, "27638": 0.10136241756387933, "insufficient_evidence": 0.014842437160909033 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.963102420879078, "probabilities": { "pearl": 0.9925448005638398, "black": 0.0035638675576000903, "edge": 0.0008489960183044536, "insufficient_evidence": 0.003042335860255638 } } }, "usage": { "input_tokens": 8213, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1489.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.49 } ], "metrics": { "requests": 15, "completed_requests": 15, "input_tokens": 118813, "output_tokens": 0, "median_latency_ms": 1444.9, "wall_s": 34.877, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "wall_s": 35.023 }