{"run_id":"run-20260512T135155-40f852","started_at":1778593915,"summary":{"run_id":"run-20260512T135155-40f852","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778593915.4604607,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7843088624338624,"score_ci_lo":0.6801421957671957,"score_ci_hi":0.8622685185185185,"top100":78,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7843088624338624,"hallu_rate_mean":0.8333333333333333,"hallu_prevalence":0.75,"hallu_top100":79,"inter_rater_agreement":0.7368551587301587,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6593,"model_cost_usd":0.589005},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8477366255144033,"score_ci_lo":0.7993827160493827,"score_ci_hi":0.897119341563786,"top100":85,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8477366255144033,"hallu_rate_mean":0.8148148148148148,"hallu_prevalence":0.8888888888888888,"hallu_top100":79,"inter_rater_agreement":0.7916666666666666,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9335,"model_cost_usd":0.7827},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6755952380952381,"score_ci_lo":0.5964285714285714,"score_ci_hi":0.7616071428571428,"top100":68,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6755952380952381,"hallu_rate_mean":2.625,"hallu_prevalence":1,"hallu_top100":34,"inter_rater_agreement":0.7977678571428571,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10769,"model_cost_usd":0.89436},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7541666666666667,"score_ci_lo":0.6458333333333334,"score_ci_hi":0.8624999999999999,"top100":75,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7541666666666667,"hallu_rate_mean":3.1666666666666665,"hallu_prevalence":1,"hallu_top100":20,"inter_rater_agreement":0.7875,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":14792,"model_cost_usd":1.23396},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5521660052910053,"score_ci_lo":0.4542824074074074,"score_ci_hi":0.6493882275132276,"top100":55,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5521660052910053,"hallu_rate_mean":0.375,"hallu_prevalence":0.5,"hallu_top100":90,"inter_rater_agreement":0.7043650793650793,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1656,"model_cost_usd":0.021022},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5601851851851852,"score_ci_lo":0.462962962962963,"score_ci_hi":0.6563786008230452,"top100":56,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5601851851851852,"hallu_rate_mean":0.25925925925925924,"hallu_prevalence":0.3333333333333333,"hallu_top100":93,"inter_rater_agreement":0.7129629629629629,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2047,"model_cost_usd":0.024398},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.24523809523809523,"score_ci_lo":0.13839285714285715,"score_ci_hi":0.3504464285714286,"top100":25,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.24523809523809523,"hallu_rate_mean":2.1666666666666665,"hallu_prevalence":1,"hallu_top100":45,"inter_rater_agreement":0.6415178571428571,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1748,"model_cost_usd":0.021442},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.49583333333333335,"score_ci_lo":0.4041666666666667,"score_ci_hi":0.5833333333333334,"top100":50,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.49583333333333335,"hallu_rate_mean":1.625,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.575,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":3031,"model_cost_usd":0.035756},{"model_id":"gpt-5","domain":"jura","score_mean":0.5980489417989417,"score_ci_lo":0.47189153439153436,"score_ci_hi":0.6937830687830687,"top100":60,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5980489417989417,"hallu_rate_mean":1.4583333333333333,"hallu_prevalence":0.875,"hallu_top100":63,"inter_rater_agreement":0.6795634920634921,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":14591,"model_cost_usd":0.235135},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8580246913580247,"score_ci_lo":0.8106995884773663,"score_ci_hi":0.911008230452675,"top100":86,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8580246913580247,"hallu_rate_mean":0.925925925925926,"hallu_prevalence":1,"hallu_top100":76,"inter_rater_agreement":0.8024691358024691,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":12483,"model_cost_usd":0.20222},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5563988095238095,"score_ci_lo":0.44687499999999997,"score_ci_hi":0.6700892857142857,"top100":56,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5563988095238095,"hallu_rate_mean":2.083333333333333,"hallu_prevalence":1,"hallu_top100":47,"inter_rater_agreement":0.7267857142857143,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":22188,"model_cost_usd":0.34782},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7458333333333333,"score_ci_lo":0.6375,"score_ci_hi":0.8458333333333333,"top100":75,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7458333333333333,"hallu_rate_mean":1.375,"hallu_prevalence":1,"hallu_top100":65,"inter_rater_agreement":0.6375,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":22344,"model_cost_usd":0.35622},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.47792658730158727,"score_ci_lo":0.44155092592592593,"score_ci_hi":0.5171957671957671,"top100":48,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.47792658730158727,"hallu_rate_mean":1.25,"hallu_prevalence":0.875,"hallu_top100":68,"inter_rater_agreement":0.5758928571428571,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2668,"model_cost_usd":0.023404},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6939300411522633,"score_ci_lo":0.6399176954732511,"score_ci_hi":0.7541152263374484,"top100":69,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6939300411522633,"hallu_rate_mean":1.1481481481481481,"hallu_prevalence":0.8888888888888888,"hallu_top100":71,"inter_rater_agreement":0.6697530864197531,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5229,"model_cost_usd":0.03776},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.16845238095238094,"score_ci_lo":0.09836309523809524,"score_ci_hi":0.24940476190476188,"top100":17,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.16845238095238094,"hallu_rate_mean":3.291666666666667,"hallu_prevalence":1,"hallu_top100":17,"inter_rater_agreement":0.6089285714285714,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3684,"model_cost_usd":0.028622},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5166666666666667,"score_ci_lo":0.39166666666666666,"score_ci_hi":0.625,"top100":52,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5166666666666667,"hallu_rate_mean":3.1666666666666665,"hallu_prevalence":1,"hallu_top100":20,"inter_rater_agreement":0.5,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6449,"model_cost_usd":0.047666}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7679453262786596,"score_ci_lo":0.7144219977553311,"score_ci_hi":0.811988936988937,"top100":77,"n_cells":33,"hallu_rate_mean_across_domains":1.8282828282828283,"hallu_top100_across_domains":54},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.4662898829565496,"score_ci_lo":0.3975509058842392,"score_ci_hi":0.5351250601250601,"top100":47,"n_cells":33,"hallu_rate_mean_across_domains":1.0808080808080807,"hallu_top100_across_domains":72},{"model_id":"gpt-5","score_mean_across_domains":0.6946809363476031,"score_ci_lo":0.6283710117043451,"score_ci_hi":0.7552068302068302,"top100":69,"n_cells":33,"hallu_rate_mean_across_domains":1.4444444444444444,"hallu_top100_across_domains":63},{"model_id":"mistral-large-2","score_mean_across_domains":0.47120410453743783,"score_ci_lo":0.4005611672278339,"score_ci_hi":0.5485730319063652,"top100":47,"n_cells":33,"hallu_rate_mean_across_domains":2.1818181818181817,"hallu_top100_across_domains":45}],"cost":{"model_calls_usd":4.8815,"judge_calls_usd":19.4993,"total_usd":24.3808,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":431320,"tokens_out_total":89959,"cost_usd":13.216725},{"judge_id":"gpt-5","tokens_in_total":224221,"tokens_out_total":273972,"cost_usd":5.230685},{"judge_id":"mistral-large-2","tokens_in_total":239552,"tokens_out_total":95469,"cost_usd":1.051918}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":886.58,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T135155-40f852/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T135155-40f852/progress.jsonl","cost_total_usd":24.3808}