{ "schema_version": 2, "project": "compaction-check", "subset": "full", "repeat": 1, "timestamp": "2026-09-24T16:06:10Z", "input_hashes": { "samples/compaction-check/fixtures/questions.json": "0d348f75b58e9df36b14babdfe9ae57f192a70d2452d757385839706712f00d1", "samples/compaction-check/fixtures/sessions.jsonl": "6b6e01855f16358971ca6734810ebec8d3a7b2240d9a8c05dbfd7c3f0ca115e6" }, "source_hashes": { "samples/_shared/community.py": "88155b9fcb87dbc2af50a59a02cdcc85a87e9404debaa6e6e3a96744c104d2cd", "samples/_shared/jev.py": "6bd98ea32afdccc886148ee92aaa1213a4ed2c9d2c797739450a1a1856d16a06", "samples/compaction-check/build_excerpts.py": "28a991f6ddcd8324b2336543bfde6f3d3ba379dd78c45a876d24d93d54d1e981", "samples/compaction-check/compact.py": "80df701cac548bf404ad02794635baab5bb7cf92b2b9e7acd92c263b6c9fd5ec" }, "label": "openjev (Codiv free hosted) - NOT JEV", "mode": "live", "provider": "codiv", "rows": [ { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "full", "context_chars": 16157, "tokens_estimate": 4040, "tokens_full_estimate": 4040, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "1": "keep", "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "16": "keep", "18": "keep", "21": "keep", "23": "keep", "25": "keep", "27": "keep", "30": "keep", "31": "keep", "33": "keep", "35": "keep", "37": "keep", "41": "keep", "43": "keep", "45": "keep", "47": "keep", "49": "keep", "51": "keep", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 4, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7849, "output_tokens": 0, "median_latency_ms": 2213.4, "wall_s": 2.214, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "last6", "context_chars": 12139, "tokens_estimate": 3035, "tokens_full_estimate": 4040, "tokens_saved_estimate": 1005, "saved_fraction": 0.24876237623762376, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "1": "drop", "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "drop", "14": "drop", "16": "drop", "18": "drop", "21": "drop", "23": "drop", "25": "drop", "27": "drop", "30": "drop", "31": "drop", "33": "drop", "35": "drop", "37": "drop", "41": "drop", "43": "drop", "45": "drop", "47": "drop", "49": "drop", "51": "drop", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 4, "abstained": 1, "answerability_correct": 4, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5447, "output_tokens": 0, "median_latency_ms": 1430.8, "wall_s": 1.431, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "trunc120", "context_chars": 12682, "tokens_estimate": 3171, "tokens_full_estimate": 4040, "tokens_saved_estimate": 869, "saved_fraction": 0.2150990099009901, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "1": "truncate", "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "truncate", "14": "truncate", "16": "truncate", "18": "truncate", "21": "truncate", "23": "truncate", "25": "truncate", "27": "truncate", "30": "truncate", "31": "truncate", "33": "truncate", "35": "truncate", "37": "truncate", "41": "truncate", "43": "truncate", "45": "truncate", "47": "truncate", "49": "truncate", "51": "truncate", "53": "keep", "56": "keep", "58": "keep", "60": "keep", "62": "keep", "64": "keep" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 4, "correct": 5, "abstained": 1, "answerability_correct": 5, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 2, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": false, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": true, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": false, "gold": "209394", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc4", "kind": "tool_dependent", "retained": true, "gold": "95586", "answer": "95586", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "yes", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 5936, "output_tokens": 0, "median_latency_ms": 1444.4, "wall_s": 2.01, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "claude-code__opus-5-5", "case_id": "claude-code__opus-5-5", "repeat": 1, "method": "model", "context_chars": 14919, "tokens_estimate": 3730, "tokens_full_estimate": 4040, "tokens_saved_estimate": 310, "saved_fraction": 0.07673267326732673, "decisions": { "keep": 18, "truncate": 0, "drop": 12 }, "result_decisions": { "1": "keep", "3": "drop", "5": "keep", "7": "keep", "9": "keep", "12": "drop", "14": "keep", "16": "drop", "18": "keep", "21": "drop", "23": "keep", "25": "drop", "27": "keep", "30": "drop", "31": "drop", "33": "keep", "35": "keep", "37": "keep", "41": "drop", "43": "keep", "45": "drop", "47": "keep", "49": "keep", "51": "keep", "53": "drop", "56": "keep", "58": "drop", "60": "keep", "62": "keep", "64": "drop" }, "unknown_result_associations": 2, "questions": 6, "answered": 6, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 4, "answered": 4, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cc1", "kind": "tool_dependent", "retained": true, "gold": "no", "answer": "no", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc2", "kind": "permanent_control", "retained": true, "gold": "commons", "answer": "commons", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc3", "kind": "tool_dependent", "retained": true, "gold": "209394", "answer": "209394", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc4", "kind": "tool_dependent", "retained": false, "gold": "95586", "answer": "95590", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cc5", "kind": "tool_dependent", "retained": true, "gold": "groups", "answer": "groups", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cc6", "kind": "permanent_control", "retained": true, "gold": "yes", "answer": "no", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 24898, "output_tokens": 0, "median_latency_ms": 2421.8, "wall_s": 5.39, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 18054, "output_tokens": 0, "median_latency_ms": 3637.5, "wall_s": 4.184, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 8117, "selector_reuse_break_even_estimated_reads": 27 }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "full", "context_chars": 9094, "tokens_estimate": 2274, "tokens_full_estimate": 2274, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 10, "truncate": 0, "drop": 0 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "keep", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4332, "output_tokens": 0, "median_latency_ms": 1306.7, "wall_s": 2.098, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "last6", "context_chars": 8362, "tokens_estimate": 2091, "tokens_full_estimate": 2274, "tokens_saved_estimate": 183, "saved_fraction": 0.08047493403693931, "decisions": { "keep": 6, "truncate": 0, "drop": 4 }, "result_decisions": { "3": "drop", "5": "drop", "7": "drop", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27638", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 3978, "output_tokens": 0, "median_latency_ms": 839.8, "wall_s": 1.516, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "trunc120", "context_chars": 8718, "tokens_estimate": 2180, "tokens_full_estimate": 2274, "tokens_saved_estimate": 94, "saved_fraction": 0.04133685136323659, "decisions": { "keep": 6, "truncate": 4, "drop": 0 }, "result_decisions": { "3": "truncate", "5": "truncate", "7": "truncate", "9": "truncate", "12": "keep", "14": "keep", "17": "keep", "19": "keep", "22": "keep", "24": "keep" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 2, "abstained": 0, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27616", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4144, "output_tokens": 0, "median_latency_ms": 785.9, "wall_s": 1.941, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "codex__gpt-6-astra", "case_id": "codex__gpt-6-astra", "repeat": 1, "method": "model", "context_chars": 9039, "tokens_estimate": 2260, "tokens_full_estimate": 2274, "tokens_saved_estimate": 14, "saved_fraction": 0.006156552330694811, "decisions": { "keep": 7, "truncate": 0, "drop": 3 }, "result_decisions": { "3": "keep", "5": "keep", "7": "keep", "9": "drop", "12": "keep", "14": "keep", "17": "keep", "19": "drop", "22": "keep", "24": "drop" }, "unknown_result_associations": 0, "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 2, "answered": 2, "retained": 2, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 1 }, "permanent_control": { "questions": 3, "answered": 3, "retained": 3, "correct": 3, "abstained": 0, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "cx1", "kind": "permanent_control", "retained": true, "gold": "reference", "answer": "reference", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx2", "kind": "permanent_control", "retained": true, "gold": "download", "answer": "download", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx3", "kind": "permanent_control", "retained": true, "gold": "script", "answer": "script", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "cx4", "kind": "tool_dependent", "retained": true, "gold": "27616", "answer": "27638", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "cx5", "kind": "tool_dependent", "retained": true, "gold": "valid", "answer": "valid", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 8927, "output_tokens": 0, "median_latency_ms": 1263.3, "wall_s": 4.631, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 4635, "output_tokens": 0, "median_latency_ms": 1096.5, "wall_s": 2.304, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 3785, "selector_reuse_break_even_estimated_reads": 271 }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "full", "context_chars": 19813, "tokens_estimate": 4954, "tokens_full_estimate": 4954, "tokens_saved_estimate": 0, "saved_fraction": 0.0, "decisions": { "keep": 30, "truncate": 0, "drop": 0 }, "result_decisions": { "4": "keep", "6": "keep", "8": "keep", "10": "keep", "13": "keep", "15": "keep", "17": "keep", "19": "keep", "22": "keep", "23": "keep", "26": "keep", "28": "keep", "30": "keep", "32": "keep", "34": "keep", "36": "keep", "39": "keep", "41": "keep", "45": "keep", "46": "keep", "48": "keep", "51": "keep", "52": "keep", "54": "keep", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 9347, "output_tokens": 0, "median_latency_ms": 1680.5, "wall_s": 2.243, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "last6", "context_chars": 15312, "tokens_estimate": 3828, "tokens_full_estimate": 4954, "tokens_saved_estimate": 1126, "saved_fraction": 0.2272910779168349, "decisions": { "keep": 6, "truncate": 0, "drop": 24 }, "result_decisions": { "4": "drop", "6": "drop", "8": "drop", "10": "drop", "13": "drop", "15": "drop", "17": "drop", "19": "drop", "22": "drop", "23": "drop", "26": "drop", "28": "drop", "30": "drop", "32": "drop", "34": "drop", "36": "drop", "39": "drop", "41": "drop", "45": "drop", "46": "drop", "48": "drop", "51": "drop", "52": "drop", "54": "drop", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 2, "correct": 2, "abstained": 2, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 3, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 1, "correct": 1, "abstained": 2, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 3 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "yes", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": false, "gold": "err", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds3", "kind": "tool_dependent", "retained": false, "gold": "1.364", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds4", "kind": "tool_dependent", "retained": false, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 6599, "output_tokens": 0, "median_latency_ms": 1211.7, "wall_s": 1.522, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "trunc120", "context_chars": 17221, "tokens_estimate": 4306, "tokens_full_estimate": 4954, "tokens_saved_estimate": 648, "saved_fraction": 0.1308033911990311, "decisions": { "keep": 6, "truncate": 24, "drop": 0 }, "result_decisions": { "4": "truncate", "6": "truncate", "8": "truncate", "10": "truncate", "13": "truncate", "15": "truncate", "17": "truncate", "19": "truncate", "22": "truncate", "23": "truncate", "26": "truncate", "28": "truncate", "30": "truncate", "32": "truncate", "34": "truncate", "36": "truncate", "39": "truncate", "41": "truncate", "45": "truncate", "46": "truncate", "48": "truncate", "51": "truncate", "52": "truncate", "54": "truncate", "56": "keep", "59": "keep", "61": "keep", "63": "keep", "65": "keep", "67": "keep" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 5, "correct": 4, "abstained": 0, "answerability_correct": 4, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": true, "gold": "auth", "answer": "auth", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "47158", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": true, "gold": "47158", "answer": "47158", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 7675, "output_tokens": 0, "median_latency_ms": 1762.9, "wall_s": 2.543, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": null, "selector_tokens_estimate": 0, "selector_reuse_break_even_estimated_reads": null }, { "session": "dsh__deepseek-flash", "case_id": "dsh__deepseek-flash", "repeat": 1, "method": "model", "context_chars": 17248, "tokens_estimate": 4312, "tokens_full_estimate": 4954, "tokens_saved_estimate": 642, "saved_fraction": 0.12959224868792896, "decisions": { "keep": 15, "truncate": 0, "drop": 15 }, "result_decisions": { "4": "keep", "6": "keep", "8": "drop", "10": "keep", "13": "drop", "15": "keep", "17": "drop", "19": "keep", "22": "drop", "23": "drop", "26": "keep", "28": "drop", "30": "keep", "32": "keep", "34": "drop", "36": "keep", "39": "keep", "41": "keep", "45": "drop", "46": "drop", "48": "drop", "51": "keep", "52": "drop", "54": "keep", "56": "drop", "59": "drop", "61": "keep", "63": "drop", "65": "keep", "67": "drop" }, "unknown_result_associations": 6, "questions": 6, "answered": 6, "retained": 4, "correct": 3, "abstained": 1, "answerability_correct": 3, "unsupported_correct_guess": 0, "lost_answer": 2, "qa_available": true, "strata": { "tool_dependent": { "questions": 5, "answered": 5, "retained": 3, "correct": 2, "abstained": 1, "answerability_correct": 2, "unsupported_correct_guess": 0, "lost_answer": 2 }, "permanent_control": { "questions": 1, "answered": 1, "retained": 1, "correct": 1, "abstained": 0, "answerability_correct": 1, "unsupported_correct_guess": 0, "lost_answer": 0 } }, "per_question": [ { "question": "ds1", "kind": "tool_dependent", "retained": false, "gold": "auth", "answer": "yes", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds2", "kind": "tool_dependent", "retained": true, "gold": "err", "answer": "err", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds3", "kind": "tool_dependent", "retained": true, "gold": "1.364", "answer": "1.364", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds4", "kind": "tool_dependent", "retained": true, "gold": "52430", "answer": "insufficient_evidence", "correct": false, "abstained": true, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": false }, { "question": "ds5", "kind": "tool_dependent", "retained": false, "gold": "47158", "answer": "52430", "correct": false, "abstained": false, "answerability_correct": false, "unsupported_correct_guess": false, "lost_answer": true }, { "question": "ds6", "kind": "permanent_control", "retained": true, "gold": "pearl", "answer": "pearl", "correct": true, "abstained": false, "answerability_correct": true, "unsupported_correct_guess": false, "lost_answer": false } ], "metrics": { "requests": 2, "completed_requests": 2, "input_tokens": 29050, "output_tokens": 0, "median_latency_ms": 2442.6, "wall_s": 5.113, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_metrics": { "requests": 1, "completed_requests": 1, "input_tokens": 20974, "output_tokens": 0, "median_latency_ms": 3574.3, "wall_s": 3.802, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "selector_tokens_estimate": 9095, "selector_reuse_break_even_estimated_reads": 15 } ], "summary": { "methods": { "full": { "sessions": 3, "tokens_estimate": 11268, "tokens_full_estimate": 11268, "saved_fraction": 0.0, "questions": 17, "answered": 17, "retained": 17, "correct": 15, "abstained": 0, "answerability_correct": 15, "unsupported_correct_guess": 0, "lost_answer": 0, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 11, "correct": 10, "abstained": 0, "answerability_correct": 10, "unsupported_correct_guess": 0, "lost_answer": 0 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "last6": { "sessions": 3, "tokens_estimate": 8954, "tokens_full_estimate": 11268, "saved_fraction": 0.20536031238906638, "questions": 17, "answered": 17, "retained": 11, "correct": 10, "abstained": 3, "answerability_correct": 12, "unsupported_correct_guess": 1, "lost_answer": 5, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 5, "correct": 5, "abstained": 3, "answerability_correct": 7, "unsupported_correct_guess": 1, "lost_answer": 5 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "trunc120": { "sessions": 3, "tokens_estimate": 9657, "tokens_full_estimate": 11268, "saved_fraction": 0.14297124600638977, "questions": 17, "answered": 17, "retained": 15, "correct": 15, "abstained": 1, "answerability_correct": 15, "unsupported_correct_guess": 1, "lost_answer": 1, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 9, "correct": 9, "abstained": 1, "answerability_correct": 9, "unsupported_correct_guess": 1, "lost_answer": 1 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 6, "abstained": 0, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 0 } } }, "model": { "sessions": 3, "tokens_estimate": 10302, "tokens_full_estimate": 11268, "saved_fraction": 0.08572949946751862, "questions": 17, "answered": 17, "retained": 14, "correct": 11, "abstained": 1, "answerability_correct": 11, "unsupported_correct_guess": 0, "lost_answer": 4, "qa_available": true, "strata": { "tool_dependent": { "questions": 11, "answered": 11, "retained": 8, "correct": 6, "abstained": 1, "answerability_correct": 6, "unsupported_correct_guess": 0, "lost_answer": 4 }, "permanent_control": { "questions": 6, "answered": 6, "retained": 6, "correct": 5, "abstained": 0, "answerability_correct": 5, "unsupported_correct_guess": 0, "lost_answer": 0 } } } } }, "policy": { "version": "compaction-v2", "methods": [ "full", "last6", "trunc120", "model" ], "keep_last_results": 6, "truncate_chars": 120, "model_decisions": [ "keep", "drop" ], "token_estimate": "ceil(rendered context characters / 4); not provider token usage", "association": "singleton balanced visible group with no history gap; otherwise unknown", "evidence": "any annotated alternative whose event-specific spans all survive", "scope": "three committed scrubbed excerpts; no full-transcript extrapolation" }, "status": "complete", "calls": [ { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:full", "state_hash": "09f53da178caf35ec8ab71f38ae731fb32bee8d62f2b4f473e617cde73e00f1d", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9967759875803526, "probabilities": { "yes": 0.00025474378994183415, "no": 0.999531569900497, "unknown": 3.0271244559279805e-05, "insufficient_evidence": 0.00018341506500194123 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.8073568763059313, "probabilities": { "commons": 0.9249415856818886, "sony": 0.00028879269644099744, "none": 0.05941284619725042, "generated": 0.0009839273344217085, "insufficient_evidence": 0.014372848089998329 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.7336722329009661, "probabilities": { "209394": 0.8716809728522082, "223434": 0.003073931409508509, "233659": 0.005735349833179691, "379231": 0.11809918203964198, "insufficient_evidence": 0.0014105638654615784 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.8241904027349323, "probabilities": { "95586": 0.9388229463079684, "95590": 0.036335626716613625, "47158": 0.0016237394728126299, "27638": 0.021760906046796608, "insufficient_evidence": 0.0014567814558087105 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9854350522094345, "probabilities": { "groups": 0.9969753463157793, "none": 0.0017106977604138041, "gloss": 0.0010283765094838224, "all": 0.0002447995105666724, "insufficient_evidence": 4.077990375635648e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.9359688874427596, "probabilities": { "yes": 0.001287391944876842, "no": 0.9877220464790483, "insufficient_evidence": 0.010990561576074852 } } }, "usage": { "input_tokens": 7849, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 2213.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.214 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:last6", "state_hash": "d92967ac2586f77e2370c343f2d044ea4cab43cbed4ae6082644954af3502493", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9699407111943976, "probabilities": { "yes": 0.0040141452641202224, "no": 0.9939566423191202, "unknown": 0.0003385547105097951, "insufficient_evidence": 0.0016906577062497372 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9867577531501317, "probabilities": { "commons": 0.9972274796344992, "sony": 0.00013303566723472826, "none": 0.0005154801527649906, "generated": 0.0020309363661186143, "insufficient_evidence": 9.30681793824786e-05 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.8775527823395081, "probabilities": { "209394": 0.00042184233970892876, "223434": 0.0066639579235367626, "233659": 0.01091129919842817, "379231": 0.018871686371761967, "insufficient_evidence": 0.9631312141665642 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.8824028450125858, "probabilities": { "95586": 0.961463581001723, "95590": 0.009407462300315783, "47158": 0.0006269498286863193, "27638": 0.028225665736348965, "insufficient_evidence": 0.00027634113292590866 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9895148315802423, "probabilities": { "groups": 0.9979674026341893, "none": 0.00108526986419355, "gloss": 0.0002915555358548454, "all": 0.0005687272022834772, "insufficient_evidence": 8.704476347891121e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.3456344411959008, "probabilities": { "yes": 0.4848521144386751, "no": 0.5105644797881042, "insufficient_evidence": 0.004583405773220814 } } }, "usage": { "input_tokens": 5447, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1430.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.431 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:trunc120", "state_hash": "308d7e00b7db233c2c64067c428d6533c53b57192d415d967f011d0d6a1a30df", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9802547890726858, "probabilities": { "yes": 0.0020750607946103962, "no": 0.996396421876627, "unknown": 0.0008645706709977396, "insufficient_evidence": 0.0006639466577649324 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.99022223393819, "probabilities": { "commons": 0.9981795688378935, "sony": 0.0003666379225546419, "none": 0.0006743518393900957, "generated": 0.0004883223427872597, "insufficient_evidence": 0.000291119057374539 } }, "cc3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9045339253982606, "probabilities": { "209394": 0.0005400582017527334, "223434": 0.010415187822666207, "233659": 0.011394096171309598, "379231": 0.004547755974656198, "insufficient_evidence": 0.9731029018296153 } }, "cc4": { "type": "choice", "choice": "95586", "confidence": 0.686001859544078, "probabilities": { "95586": 0.8398428950725394, "95590": 0.0123927867488134, "47158": 0.0024675159480049896, "27638": 0.14359243523407092, "insufficient_evidence": 0.0017043669965713104 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9618048773633331, "probabilities": { "groups": 0.9907721316739342, "none": 0.005796506664827675, "gloss": 0.0020388816692936966, "all": 0.0011504626687590734, "insufficient_evidence": 0.00024201732318544534 } }, "cc6": { "type": "choice", "choice": "yes", "confidence": 0.7150966266344505, "probabilities": { "yes": 0.9148369726107068, "no": 0.0791256202418244, "insufficient_evidence": 0.006037407147468829 } } }, "usage": { "input_tokens": 5936, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1444.4, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.01 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "select", "state_hash": "1ab156da003911d9ea375ce54aacbf19201cc6adba8bc73315f688d4ed39937a", "questions_hash": "b934e72498fb1f126d8c79bf9a0c69e5e573708edd8658a1f5e2ae40d4d4cb83", "question_ids": [ "result_1", "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_16", "result_18", "result_21", "result_23", "result_25", "result_27", "result_30", "result_31", "result_33", "result_35", "result_37", "result_41", "result_43", "result_45", "result_47", "result_49", "result_51", "result_53", "result_56", "result_58", "result_60", "result_62", "result_64" ], "answers": { "result_1": { "type": "choice", "choice": "keep", "confidence": 0.8612663946324992, "probabilities": { "keep": 0.9804805892722542, "drop": 0.019519410727745688 } }, "result_3": { "type": "choice", "choice": "drop", "confidence": 0.9796287795720037, "probabilities": { "keep": 0.0019506895020530326, "drop": 0.998049310497947 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.7338457500377056, "probabilities": { "keep": 0.9546844282063522, "drop": 0.04531557179364777 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.22411671327288096, "probabilities": { "keep": 0.7712172694448471, "drop": 0.22878273055515277 } }, "result_9": { "type": "choice", "choice": "keep", "confidence": 0.5991426908190465, "probabilities": { "keep": 0.9203747635359373, "drop": 0.07962523646406268 } }, "result_12": { "type": "choice", "choice": "drop", "confidence": 0.10888818690865487, "probabilities": { "keep": 0.30822321955532556, "drop": 0.6917767804446744 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.10262062279363038, "probabilities": { "keep": 0.6863168462911217, "drop": 0.3136831537088784 } }, "result_16": { "type": "choice", "choice": "drop", "confidence": 0.11161325892239404, "probabilities": { "keep": 0.3059024163769912, "drop": 0.6940975836230088 } }, "result_18": { "type": "choice", "choice": "keep", "confidence": 0.4100687027595119, "probabilities": { "keep": 0.8578155215624705, "drop": 0.14218447843752946 } }, "result_21": { "type": "choice", "choice": "drop", "confidence": 0.4718954313125088, "probabilities": { "keep": 0.11956340003284882, "drop": 0.8804365999671512 } }, "result_23": { "type": "choice", "choice": "keep", "confidence": 0.31722415611529853, "probabilities": { "keep": 0.8187629225762717, "drop": 0.18123707742372827 } }, "result_25": { "type": "choice", "choice": "drop", "confidence": 0.02025420948169021, "probabilities": { "keep": 0.4164137423107232, "drop": 0.5835862576892769 } }, "result_27": { "type": "choice", "choice": "keep", "confidence": 0.29628303356734476, "probabilities": { "keep": 0.8089239919466318, "drop": 0.19107600805336813 } }, "result_30": { "type": "choice", "choice": "drop", "confidence": 0.2001983438377356, "probabilities": { "keep": 0.24288289630213186, "drop": 0.7571171036978681 } }, "result_31": { "type": "choice", "choice": "drop", "confidence": 0.08771431421319631, "probabilities": { "keep": 0.3274364003725462, "drop": 0.6725635996274537 } }, "result_33": { "type": "choice", "choice": "keep", "confidence": 0.03650209021906903, "probabilities": { "keep": 0.611998224077064, "drop": 0.38800177592293605 } }, "result_35": { "type": "choice", "choice": "keep", "confidence": 0.32782119669291854, "probabilities": { "keep": 0.8235816078771063, "drop": 0.17641839212289367 } }, "result_37": { "type": "choice", "choice": "keep", "confidence": 0.2608650299721301, "probabilities": { "keep": 0.7912248403539566, "drop": 0.20877515964604348 } }, "result_41": { "type": "choice", "choice": "drop", "confidence": 0.0018015755910404385, "probabilities": { "keep": 0.4750176346512738, "drop": 0.5249823653487262 } }, "result_43": { "type": "choice", "choice": "keep", "confidence": 0.09757844351818357, "probabilities": { "keep": 0.6817924656123039, "drop": 0.31820753438769606 } }, "result_45": { "type": "choice", "choice": "drop", "confidence": 0.005585983117401105, "probabilities": { "keep": 0.456028937710877, "drop": 0.543971062289123 } }, "result_47": { "type": "choice", "choice": "keep", "confidence": 0.02620922400468806, "probabilities": { "keep": 0.5950172848753539, "drop": 0.4049827151246461 } }, "result_49": { "type": "choice", "choice": "keep", "confidence": 0.016719849032680756, "probabilities": { "keep": 0.5759752453208092, "drop": 0.42402475467919076 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.0028670219383044593, "probabilities": { "keep": 0.5315115170399669, "drop": 0.46848848296003315 } }, "result_53": { "type": "choice", "choice": "drop", "confidence": 0.004978681560935838, "probabilities": { "keep": 0.4584850174615983, "drop": 0.5415149825384017 } }, "result_56": { "type": "choice", "choice": "keep", "confidence": 0.005787130542842744, "probabilities": { "keep": 0.5447547022427661, "drop": 0.4552452977572339 } }, "result_58": { "type": "choice", "choice": "drop", "confidence": 0.018265749123672514, "probabilities": { "keep": 0.42060437913796656, "drop": 0.5793956208620334 } }, "result_60": { "type": "choice", "choice": "keep", "confidence": 0.0124339073369365, "probabilities": { "keep": 0.5655505131210296, "drop": 0.43444948687897045 } }, "result_62": { "type": "choice", "choice": "keep", "confidence": 0.12556436284788253, "probabilities": { "keep": 0.7055220777991289, "drop": 0.29447792220087105 } }, "result_64": { "type": "choice", "choice": "drop", "confidence": 0.0822236207464827, "probabilities": { "keep": 0.3328147517128512, "drop": 0.6671852482871488 } } }, "usage": { "input_tokens": 18054, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 3637.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 4.184 }, { "case_id": "claude-code__opus-5-5:repeat-1", "phase": "reader:model", "state_hash": "9760a1a1a22a37a29369064ff3caa91dcb765b1f8970ae69ee8f0548cd0b9737", "questions_hash": "12dbdf65868f2fc731bd3ac75d93685c0d636216a34603162129bfc35c820050", "question_ids": [ "cc1", "cc2", "cc3", "cc4", "cc5", "cc6" ], "answers": { "cc1": { "type": "choice", "choice": "no", "confidence": 0.9972169638689263, "probabilities": { "yes": 0.0002787218020335551, "no": 0.9995995068333524, "unknown": 4.240090291803964e-05, "insufficient_evidence": 7.937046169597261e-05 } }, "cc2": { "type": "choice", "choice": "commons", "confidence": 0.9916187754288631, "probabilities": { "commons": 0.998471965802929, "sony": 0.0002382178477984567, "none": 0.000546785987183865, "generated": 0.00038703529044645, "insufficient_evidence": 0.00035599507164227253 } }, "cc3": { "type": "choice", "choice": "209394", "confidence": 0.4705017274268073, "probabilities": { "209394": 0.7150753813665454, "223434": 0.007912663365722059, "233659": 0.1264674924553182, "379231": 0.1438024156523276, "insufficient_evidence": 0.006742047160086751 } }, "cc4": { "type": "choice", "choice": "95590", "confidence": 0.5599919107382545, "probabilities": { "95586": 0.30514155357522244, "95590": 0.678854394048673, "47158": 0.003165636902347118, "27638": 0.006760200924742994, "insufficient_evidence": 0.006078214549014477 } }, "cc5": { "type": "choice", "choice": "groups", "confidence": 0.9946080012297612, "probabilities": { "groups": 0.9990330659062092, "none": 0.0005409256032473366, "gloss": 0.00014671270584997814, "all": 0.00026301466525977034, "insufficient_evidence": 1.6281119433647487e-05 } }, "cc6": { "type": "choice", "choice": "no", "confidence": 0.5376435147167811, "probabilities": { "yes": 0.13563180904722924, "no": 0.8399966589789262, "insufficient_evidence": 0.024371531973844583 } } }, "usage": { "input_tokens": 6844, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1206.0, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.206 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:full", "state_hash": "818641e42779c98df892775aec9b311b7dd75a0f45931036724b66d247b82d59", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.999011457898129, "probabilities": { "reference": 0.999863845307214, "render": 0.0001317606210862287, "none": 1.5372794181860918e-06, "insufficient_evidence": 2.856792281782902e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9996247772848365, "probabilities": { "download": 0.9999561184466199, "drawn": 2.8199835035421223e-05, "omitted": 2.811019310287047e-06, "insufficient_evidence": 1.2870699034274217e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9899884347370689, "probabilities": { "script": 0.9982896690488904, "patch": 0.0004029673764721383, "echo": 0.001212695943275328, "insufficient_evidence": 9.4667631362094e-05 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.9619033990741891, "probabilities": { "27616": 0.9890654300189754, "27638": 0.010753489013550001, "47158": 2.9845002472462776e-05, "95586": 0.00013193662420146653, "insufficient_evidence": 1.9299340800740533e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9966311188967338, "probabilities": { "valid": 0.9995134359458079, "invalid": 0.00028261940003311494, "notrun": 0.0001019723270795428, "insufficient_evidence": 0.0001019723270795428 } } }, "usage": { "input_tokens": 4332, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1306.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.098 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:last6", "state_hash": "37c153d6143d560dec7b586528ae5881e640484c9fb38cbdb6b22ac0ed8ce18b", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9992415350887298, "probabilities": { "reference": 0.999898834027145, "render": 9.702832864678196e-05, "none": 1.119945832364072e-06, "insufficient_evidence": 3.0176983759044695e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9991051893084945, "probabilities": { "download": 0.9998858617418537, "drawn": 3.179780137878572e-05, "omitted": 6.465166233230782e-06, "insufficient_evidence": 7.587529053427253e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9716096555375299, "probabilities": { "script": 0.9940302030804238, "patch": 0.0004875397987199924, "echo": 0.005188492821428891, "insufficient_evidence": 0.00029376429942726723 } }, "cx4": { "type": "choice", "choice": "27638", "confidence": 0.8581600818738927, "probabilities": { "27616": 0.05798755842026996, "27638": 0.941265756883078, "47158": 0.00020245415960760317, "95586": 0.0002445825507119486, "insufficient_evidence": 0.00029964798633248964 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9966419601595183, "probabilities": { "valid": 0.9995049601791417, "invalid": 0.0003530826567269248, "notrun": 8.401373023929572e-05, "insufficient_evidence": 5.7943433892139866e-05 } } }, "usage": { "input_tokens": 3978, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 839.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.516 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:trunc120", "state_hash": "f3efcd2121b75907229d1975779876597872e4ff72d0dd63699692848b9fae3f", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9994175615091061, "probabilities": { "reference": 0.9999242082737491, "render": 7.319130945760429e-05, "none": 4.788426355201996e-07, "insufficient_evidence": 2.1215741578668577e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9988432698093878, "probabilities": { "download": 0.9998453727062584, "drawn": 2.617763684387653e-05, "omitted": 5.228048069781585e-06, "insufficient_evidence": 0.00012322160882782772 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9502343195063995, "probabilities": { "script": 0.9876236701822139, "patch": 0.0002984764870159042, "echo": 0.011899854194309393, "insufficient_evidence": 0.00017799913646088412 } }, "cx4": { "type": "choice", "choice": "27616", "confidence": 0.9881789452264144, "probabilities": { "27616": 0.9973999407576987, "27638": 0.0023516973781988844, "47158": 8.930731768171614e-06, "95586": 0.00020718080164601552, "insufficient_evidence": 3.2250330688218634e-05 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9992686530561574, "probabilities": { "valid": 0.999909410377457, "invalid": 4.800976007349457e-05, "notrun": 6.010862736556106e-06, "insufficient_evidence": 3.6568999733002504e-05 } } }, "usage": { "input_tokens": 4144, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 785.9, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.941 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "select", "state_hash": "f3b419d113ea7359abd8053f120ba9422237f5cf245aba011093c383b4fc09ec", "questions_hash": "9cac56873501140b02d037d69f22e218df2321487095362f67a020b3f35c89ef", "question_ids": [ "result_3", "result_5", "result_7", "result_9", "result_12", "result_14", "result_17", "result_19", "result_22", "result_24" ], "answers": { "result_3": { "type": "choice", "choice": "keep", "confidence": 0.17920038816939776, "probabilities": { "keep": 0.7439028570873959, "drop": 0.2560971429126041 } }, "result_5": { "type": "choice", "choice": "keep", "confidence": 0.35511442482100153, "probabilities": { "keep": 0.8355347739176134, "drop": 0.16446522608238665 } }, "result_7": { "type": "choice", "choice": "keep", "confidence": 0.00018835532622052398, "probabilities": { "keep": 0.5080793651180151, "drop": 0.49192063488198495 } }, "result_9": { "type": "choice", "choice": "drop", "confidence": 0.26594079522852454, "probabilities": { "keep": 0.20615017492109805, "drop": 0.7938498250789019 } }, "result_12": { "type": "choice", "choice": "keep", "confidence": 0.3705633634776234, "probabilities": { "keep": 0.8420276472433149, "drop": 0.15797235275668503 } }, "result_14": { "type": "choice", "choice": "keep", "confidence": 0.246262245101598, "probabilities": { "keep": 0.7834930792863913, "drop": 0.2165069207136086 } }, "result_17": { "type": "choice", "choice": "keep", "confidence": 0.025828078788044828, "probabilities": { "keep": 0.5943280635338603, "drop": 0.40567193646613975 } }, "result_19": { "type": "choice", "choice": "drop", "confidence": 0.5636306419029089, "probabilities": { "keep": 0.08996990565832344, "drop": 0.9100300943416766 } }, "result_22": { "type": "choice", "choice": "keep", "confidence": 0.7010327727809254, "probabilities": { "keep": 0.9470094605377549, "drop": 0.052990539462245134 } }, "result_24": { "type": "choice", "choice": "drop", "confidence": 0.00013998507487100742, "probabilities": { "keep": 0.4930348323194519, "drop": 0.5069651676805481 } } }, "usage": { "input_tokens": 4635, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1096.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.304 }, { "case_id": "codex__gpt-6-astra:repeat-1", "phase": "reader:model", "state_hash": "1553b949a602c04972797b66649c4132021c0f171a43c4a8772135ea1cc83af8", "questions_hash": "30fe63c00567a4071ec91dfaa59ed12a2c39b865a6f4076806eb5c2c7094c7ef", "question_ids": [ "cx1", "cx2", "cx3", "cx4", "cx5" ], "answers": { "cx1": { "type": "choice", "choice": "reference", "confidence": 0.9992125762172208, "probabilities": { "reference": 0.9998965701533539, "render": 9.282363993463588e-05, "none": 1.807723615837966e-06, "insufficient_evidence": 8.798483095544385e-06 } }, "cx2": { "type": "choice", "choice": "download", "confidence": 0.9995783970400376, "probabilities": { "download": 0.9999502388796115, "drawn": 2.9528601478685545e-05, "omitted": 2.7491769212597777e-06, "insufficient_evidence": 1.748334198850543e-05 } }, "cx3": { "type": "choice", "choice": "script", "confidence": 0.9857794972939931, "probabilities": { "script": 0.997361763420093, "patch": 0.0002491383701558436, "echo": 0.00222828380194904, "insufficient_evidence": 0.00016081440780212208 } }, "cx4": { "type": "choice", "choice": "27638", "confidence": 0.9443283407609661, "probabilities": { "27616": 0.015863215705198955, "27638": 0.9832399255593621, "47158": 0.00019711001676188348, "95586": 0.0003218943490177164, "insufficient_evidence": 0.0003778543696593124 } }, "cx5": { "type": "choice", "choice": "valid", "confidence": 0.9978210053214182, "probabilities": { "valid": 0.999690882598884, "invalid": 0.00023638053077747, "notrun": 2.2237969953156592e-05, "insufficient_evidence": 5.049890038540036e-05 } } }, "usage": { "input_tokens": 4292, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1430.1, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.327 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:full", "state_hash": "2c718871c891948a99e03c50664a5cacbfa33a885c063e9ae0760a8b166fb44a", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9979247856746383, "probabilities": { "yes": 0.00012541133453593752, "auth": 0.9997119231445868, "timeout": 1.2097734133342844e-05, "insufficient_evidence": 0.00015056778674393223 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.98903491888471, "probabilities": { "err": 0.9981886182548505, "ok": 0.000648871766721644, "missing": 0.0007188444161300746, "insufficient_evidence": 0.00044366556229786386 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9971838016816601, "probabilities": { "1.364": 0.9995198118320345, "1.000": 0.00035085657639525843, "1.455": 3.3829769929882996e-05, "0.733": 8.683272436315458e-05, "insufficient_evidence": 8.669097277196013e-06 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.8629745108661482, "probabilities": { "52430": 0.03000643377672494, "47158": 0.9555783073426044, "95586": 0.0018763859711987978, "27616": 0.01070758209110279, "insufficient_evidence": 0.0018312908183690715 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.9378004257390781, "probabilities": { "47158": 0.9812090214109968, "52430": 0.017148900688409113, "95586": 0.0005284818244048888, "27638": 0.0010621639488996176, "insufficient_evidence": 5.143212728964658e-05 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9768700274317498, "probabilities": { "pearl": 0.9954742140202085, "black": 0.0034689989085113, "edge": 0.00033458934845374415, "insufficient_evidence": 0.0007221977228264005 } } }, "usage": { "input_tokens": 9347, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1680.5, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.243 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:last6", "state_hash": "f746af707c79a9b701f3828e24bbb0a2f7b6e6f79d063fdc225ec7a0a07a57c9", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "yes", "confidence": 0.9937625870856793, "probabilities": { "yes": 0.9989946765571629, "auth": 0.0005862932218697312, "timeout": 3.456179495495139e-06, "insufficient_evidence": 0.00041557404147188397 } }, "ds2": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.9682125277169886, "probabilities": { "err": 0.0022168145155644706, "ok": 0.0037633335288940317, "missing": 0.0004026018740885059, "insufficient_evidence": 0.9936172500814531 } }, "ds3": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.7712536845886238, "probabilities": { "1.364": 0.014223211150842224, "1.000": 0.024464176205370187, "1.455": 0.006529274547495849, "0.733": 0.031921475821143165, "insufficient_evidence": 0.9228618622751485 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.5692529624386538, "probabilities": { "52430": 0.16151827690689943, "47158": 0.7767897725465976, "95586": 0.0024457228038876834, "27616": 0.052769751946916646, "insufficient_evidence": 0.0064764757956986075 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.7151834114989277, "probabilities": { "47158": 0.8391670832357003, "52430": 0.1577624044775863, "95586": 0.0003745035137605695, "27638": 0.002361039774952653, "insufficient_evidence": 0.0003349689980001464 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9774709715429893, "probabilities": { "pearl": 0.9958103582237319, "black": 0.0021549483550778736, "edge": 0.0005732734381035398, "insufficient_evidence": 0.0014614199830865647 } } }, "usage": { "input_tokens": 6599, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1211.7, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.522 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:trunc120", "state_hash": "637b10b060d11f04ea8ac550b6678f62ddfc5a4b1ef52ebda9f99a98a5dea062", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "auth", "confidence": 0.9983870616327707, "probabilities": { "yes": 0.00014912897881742677, "auth": 0.9997789822623104, "timeout": 3.927765919481067e-06, "insufficient_evidence": 6.796099295267449e-05 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.992153207944262, "probabilities": { "err": 0.9987628075016928, "ok": 0.00039516798777083873, "missing": 0.0004227959965592714, "insufficient_evidence": 0.0004192285139771345 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9944129886137335, "probabilities": { "1.364": 0.9990043184313221, "1.000": 0.0004891669797271009, "1.455": 0.00028546744741080136, "0.733": 0.00019627619054745043, "insufficient_evidence": 2.4770950992554353e-05 } }, "ds4": { "type": "choice", "choice": "47158", "confidence": 0.8895167296885668, "probabilities": { "52430": 0.03684057017083278, "47158": 0.9605002309116828, "95586": 0.00033298827666544635, "27616": 0.0021102391378507953, "insufficient_evidence": 0.00021597150296819882 } }, "ds5": { "type": "choice", "choice": "47158", "confidence": 0.8431529423793253, "probabilities": { "47158": 0.9373378592317618, "52430": 0.05861030500454496, "95586": 0.0023033717132729076, "27638": 0.0016348385740679457, "insufficient_evidence": 0.00011362547635230406 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9923573890344037, "probabilities": { "pearl": 0.998746661347602, "black": 0.0008936279594301483, "edge": 0.00010399066857887343, "insufficient_evidence": 0.00025572002438896337 } } }, "usage": { "input_tokens": 7675, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1762.9, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 2.543 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "select", "state_hash": "6d057db834e25182a5101a4d62be8256aafc4528a6356d6b34f5d0f2becfacc5", "questions_hash": "7f2e216b1bdff1cdf9c33d7a6e5296b61b3ef74b7ce862303c290a1473566dd9", "question_ids": [ "result_4", "result_6", "result_8", "result_10", "result_13", "result_15", "result_17", "result_19", "result_22", "result_23", "result_26", "result_28", "result_30", "result_32", "result_34", "result_36", "result_39", "result_41", "result_45", "result_46", "result_48", "result_51", "result_52", "result_54", "result_56", "result_59", "result_61", "result_63", "result_65", "result_67" ], "answers": { "result_4": { "type": "choice", "choice": "keep", "confidence": 0.5672348730995569, "probabilities": { "keep": 0.9111066530152464, "drop": 0.0888933469847536 } }, "result_6": { "type": "choice", "choice": "keep", "confidence": 0.0027042627190728696, "probabilities": { "keep": 0.5306045796898168, "drop": 0.4693954203101832 } }, "result_8": { "type": "choice", "choice": "drop", "confidence": 0.13654332745916442, "probabilities": { "keep": 0.2859687116082381, "drop": 0.7140312883917619 } }, "result_10": { "type": "choice", "choice": "keep", "confidence": 0.2458410413121046, "probabilities": { "keep": 0.783265959150885, "drop": 0.2167340408491149 } }, "result_13": { "type": "choice", "choice": "drop", "confidence": 0.03876459184970038, "probabilities": { "keep": 0.3846136199602253, "drop": 0.6153863800397746 } }, "result_15": { "type": "choice", "choice": "keep", "confidence": 0.05037152321569116, "probabilities": { "keep": 0.6313518131166649, "drop": 0.36864818688333506 } }, "result_17": { "type": "choice", "choice": "drop", "confidence": 0.07488869211497262, "probabilities": { "keep": 0.3403060868051384, "drop": 0.6596939131948616 } }, "result_19": { "type": "choice", "choice": "keep", "confidence": 0.22317747707949664, "probabilities": { "keep": 0.7706808595735406, "drop": 0.22931914042645932 } }, "result_22": { "type": "choice", "choice": "drop", "confidence": 0.04000905627197926, "probabilities": { "keep": 0.38279325317794793, "drop": 0.6172067468220521 } }, "result_23": { "type": "choice", "choice": "drop", "confidence": 0.019203581677550874, "probabilities": { "keep": 0.4186005444020625, "drop": 0.5813994555979375 } }, "result_26": { "type": "choice", "choice": "keep", "confidence": 0.3802058256755454, "probabilities": { "keep": 0.8459862728008989, "drop": 0.15401372719910117 } }, "result_28": { "type": "choice", "choice": "drop", "confidence": 0.005280205120900883, "probabilities": { "keep": 0.45724785380854105, "drop": 0.542752146191459 } }, "result_30": { "type": "choice", "choice": "keep", "confidence": 0.09604604428617192, "probabilities": { "keep": 0.6803926486477024, "drop": 0.3196073513522976 } }, "result_32": { "type": "choice", "choice": "keep", "confidence": 0.141185556038385, "probabilities": { "keep": 0.7175152210834007, "drop": 0.28248477891659934 } }, "result_34": { "type": "choice", "choice": "drop", "confidence": 0.021850607796379817, "probabilities": { "keep": 0.4131983081408892, "drop": 0.5868016918591108 } }, "result_36": { "type": "choice", "choice": "keep", "confidence": 0.05095016955367371, "probabilities": { "keep": 0.6320950931868153, "drop": 0.3679049068131846 } }, "result_39": { "type": "choice", "choice": "keep", "confidence": 0.29027853157880446, "probabilities": { "keep": 0.8060209681941534, "drop": 0.1939790318058466 } }, "result_41": { "type": "choice", "choice": "keep", "confidence": 0.6320949236141165, "probabilities": { "keep": 0.9294682057560495, "drop": 0.07053179424395051 } }, "result_45": { "type": "choice", "choice": "drop", "confidence": 0.45603773385543134, "probabilities": { "keep": 0.12514175458358012, "drop": 0.8748582454164199 } }, "result_46": { "type": "choice", "choice": "drop", "confidence": 0.2536277025129018, "probabilities": { "keep": 0.21257303305322137, "drop": 0.7874269669467786 } }, "result_48": { "type": "choice", "choice": "drop", "confidence": 0.0019141314616446525, "probabilities": { "keep": 0.47424938889240675, "drop": 0.5257506111075932 } }, "result_51": { "type": "choice", "choice": "keep", "confidence": 0.007558031909144058, "probabilities": { "keep": 0.5511354752433961, "drop": 0.4488645247566039 } }, "result_52": { "type": "choice", "choice": "drop", "confidence": 9.928728633379347e-05, "probabilities": { "keep": 0.4941340335915497, "drop": 0.5058659664084503 } }, "result_54": { "type": "choice", "choice": "keep", "confidence": 0.1779414175590318, "probabilities": { "keep": 0.7430828519549433, "drop": 0.25691714804505666 } }, "result_56": { "type": "choice", "choice": "drop", "confidence": 0.3885527692914985, "probabilities": { "keep": 0.15064306198496547, "drop": 0.8493569380150345 } }, "result_59": { "type": "choice", "choice": "drop", "confidence": 0.034716619814020744, "probabilities": { "keep": 0.39075239634126013, "drop": 0.6092476036587399 } }, "result_61": { "type": "choice", "choice": "keep", "confidence": 0.0002619640345812302, "probabilities": { "keep": 0.5095280806389433, "drop": 0.49047191936105666 } }, "result_63": { "type": "choice", "choice": "drop", "confidence": 0.07700454301946713, "probabilities": { "keep": 0.3381067446576465, "drop": 0.6618932553423534 } }, "result_65": { "type": "choice", "choice": "keep", "confidence": 0.06928361279576711, "probabilities": { "keep": 0.6537041189792459, "drop": 0.3462958810207542 } }, "result_67": { "type": "choice", "choice": "drop", "confidence": 1.070306310246849e-08, "probabilities": { "keep": 0.499939095164407, "drop": 0.5000609048355931 } } }, "usage": { "input_tokens": 20974, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 3574.3, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 3.802 }, { "case_id": "dsh__deepseek-flash:repeat-1", "phase": "reader:model", "state_hash": "476c759d7b1b68a53066fa2251ea68620d08910e00803981eb2e0e3d4cbaee83", "questions_hash": "6211342c34ab74e38a868da0e17a4274c32e63ac44f3868a672bfe788a5b6df5", "question_ids": [ "ds1", "ds2", "ds3", "ds4", "ds5", "ds6" ], "answers": { "ds1": { "type": "choice", "choice": "yes", "confidence": 0.9861497689649517, "probabilities": { "yes": 0.9975040270158746, "auth": 0.0015317062494160268, "timeout": 1.664739554380333e-05, "insufficient_evidence": 0.0009476193391654917 } }, "ds2": { "type": "choice", "choice": "err", "confidence": 0.7121623033541258, "probabilities": { "err": 0.8677306210951099, "ok": 0.0009665108040092099, "missing": 0.13091738366362426, "insufficient_evidence": 0.00038548443725670275 } }, "ds3": { "type": "choice", "choice": "1.364", "confidence": 0.9866790459355647, "probabilities": { "1.364": 0.9973415188528238, "1.000": 0.0009931825935901371, "1.455": 0.0008642405808517325, "0.733": 0.0007765507013872385, "insufficient_evidence": 2.450727134703628e-05 } }, "ds4": { "type": "choice", "choice": "insufficient_evidence", "confidence": 0.34501465172662915, "probabilities": { "52430": 0.018015425131161694, "47158": 0.11222089458872121, "95586": 0.009327515386042061, "27616": 0.31181537436469714, "insufficient_evidence": 0.5486207905293778 } }, "ds5": { "type": "choice", "choice": "52430", "confidence": 0.5151887503807782, "probabilities": { "47158": 0.11558261862224928, "52430": 0.7777223031121643, "95586": 0.013055531961971054, "27638": 0.06583458802537712, "insufficient_evidence": 0.027804958278238284 } }, "ds6": { "type": "choice", "choice": "pearl", "confidence": 0.9788295576441531, "probabilities": { "pearl": 0.9960795194067363, "black": 0.0018373903481859544, "edge": 0.00038728816924747336, "insufficient_evidence": 0.001695802075830216 } } }, "usage": { "input_tokens": 8076, "output_tokens": 0 }, "model": "openjev-0.1", "mode": "live", "provider": "codiv", "label": "openjev (Codiv free hosted) - NOT JEV", "latency_ms": 1310.8, "cached": false, "reported_cost_usd": null, "status": "ok", "elapsed_s": 1.311 } ], "metrics": { "requests": 15, "completed_requests": 15, "input_tokens": 118182, "output_tokens": 0, "median_latency_ms": 1430.1, "wall_s": 32.652, "cost_usd": 0.0, "cost_basis": "free route; missing provider cost fields are not receipts", "reported_cost_usd": null }, "wall_s": 32.776 }