{"run_id":"run-20260516T200347-bc6991","started_at":1778961827,"summary":{"run_id":"run-20260516T200347-bc6991","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778961827.9721866,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7853835978835979,"score_ci_lo":0.6767526455026456,"score_ci_hi":0.8831018518518519,"top100":79,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7853835978835979,"hallu_rate_mean":0.9583333333333334,"hallu_prevalence":1,"hallu_top100":76,"inter_rater_agreement":0.7604166666666666,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6722,"model_cost_usd":0.59868},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.867798353909465,"score_ci_lo":0.7926954732510288,"score_ci_hi":0.9398148148148149,"top100":87,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.867798353909465,"hallu_rate_mean":1.0370370370370372,"hallu_prevalence":0.8888888888888888,"hallu_top100":74,"inter_rater_agreement":0.8796296296296295,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":10077,"model_cost_usd":0.83835},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6447916666666667,"score_ci_lo":0.5770833333333333,"score_ci_hi":0.7229166666666667,"top100":64,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6447916666666667,"hallu_rate_mean":2.2916666666666665,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.7714285714285714,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10576,"model_cost_usd":0.879885},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7416666666666666,"score_ci_lo":0.6333333333333333,"score_ci_hi":0.8416666666666666,"top100":74,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7416666666666666,"hallu_rate_mean":3,"hallu_prevalence":1,"hallu_top100":25,"inter_rater_agreement":0.6375,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":15125,"model_cost_usd":1.258935},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5408399470899471,"score_ci_lo":0.4418816137566138,"score_ci_hi":0.634920634920635,"top100":54,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5408399470899471,"hallu_rate_mean":0.20833333333333331,"hallu_prevalence":0.25,"hallu_top100":94,"inter_rater_agreement":0.7455357142857143,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1685,"model_cost_usd":0.021312},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5205761316872428,"score_ci_lo":0.4104938271604938,"score_ci_hi":0.6157407407407407,"top100":52,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5205761316872428,"hallu_rate_mean":0.6296296296296297,"hallu_prevalence":0.6666666666666666,"hallu_top100":84,"inter_rater_agreement":0.6790123456790123,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2085,"model_cost_usd":0.024778},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.29345238095238096,"score_ci_lo":0.16458333333333333,"score_ci_hi":0.42857142857142855,"top100":29,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.29345238095238096,"hallu_rate_mean":1.8333333333333333,"hallu_prevalence":0.875,"hallu_top100":54,"inter_rater_agreement":0.6433035714285714,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1755,"model_cost_usd":0.021513},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.4041666666666667,"score_ci_lo":0.2916666666666667,"score_ci_hi":0.5125,"top100":40,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4041666666666667,"hallu_rate_mean":1.6666666666666667,"hallu_prevalence":1,"hallu_top100":58,"inter_rater_agreement":0.5125,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":2837,"model_cost_usd":0.033816},{"model_id":"gpt-5","domain":"jura","score_mean":0.7414021164021164,"score_ci_lo":0.6322751322751323,"score_ci_hi":0.8391203703703703,"top100":74,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7414021164021164,"hallu_rate_mean":0.7916666666666666,"hallu_prevalence":0.625,"hallu_top100":80,"inter_rater_agreement":0.7668650793650793,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":13424,"model_cost_usd":0.21763},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8348765432098766,"score_ci_lo":0.772119341563786,"score_ci_hi":0.8940329218106995,"top100":83,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8348765432098766,"hallu_rate_mean":1.0370370370370372,"hallu_prevalence":1,"hallu_top100":74,"inter_rater_agreement":0.8518518518518519,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":13121,"model_cost_usd":0.21179},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.4617559523809524,"score_ci_lo":0.2604166666666667,"score_ci_hi":0.6547619047619048,"top100":46,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4617559523809524,"hallu_rate_mean":2.4166666666666665,"hallu_prevalence":1,"hallu_top100":39,"inter_rater_agreement":0.6477678571428571,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":22244,"model_cost_usd":0.34866},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7833333333333333,"score_ci_lo":0.7166666666666667,"score_ci_hi":0.85,"top100":78,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7833333333333333,"hallu_rate_mean":1.9583333333333333,"hallu_prevalence":1,"hallu_top100":51,"inter_rater_agreement":0.7125,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":22501,"model_cost_usd":0.358575},{"model_id":"grok-4.3","domain":"jura","score_mean":0.499917328042328,"score_ci_lo":0.421875,"score_ci_hi":0.5671296296296297,"top100":50,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.499917328042328,"hallu_rate_mean":0.375,"hallu_prevalence":0.625,"hallu_top100":90,"inter_rater_agreement":0.6488095238095238,"n_judges":3,"tokens_in_total":4643,"tokens_out_total":1071,"model_cost_usd":0.008481},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5853909465020577,"score_ci_lo":0.4861111111111111,"score_ci_hi":0.6856995884773663,"top100":59,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5853909465020577,"hallu_rate_mean":0.7037037037037037,"hallu_prevalence":0.4444444444444444,"hallu_top100":82,"inter_rater_agreement":0.7345679012345678,"n_judges":3,"tokens_in_total":4263,"tokens_out_total":2155,"model_cost_usd":0.010716},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.17857142857142855,"score_ci_lo":0.041666666666666664,"score_ci_hi":0.353422619047619,"top100":18,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.17857142857142855,"hallu_rate_mean":3.2708333333333335,"hallu_prevalence":1,"hallu_top100":18,"inter_rater_agreement":0.5580357142857143,"n_judges":3,"tokens_in_total":4343,"tokens_out_total":2240,"model_cost_usd":0.011029},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.4125,"score_ci_lo":0.24166666666666667,"score_ci_hi":0.5791666666666667,"top100":41,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4125,"hallu_rate_mean":2,"hallu_prevalence":1,"hallu_top100":50,"inter_rater_agreement":0.55,"n_judges":3,"tokens_in_total":5722,"tokens_out_total":3400,"model_cost_usd":0.015652},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4728835978835979,"score_ci_lo":0.38095238095238093,"score_ci_hi":0.578125,"top100":47,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4728835978835979,"hallu_rate_mean":1.5416666666666667,"hallu_prevalence":0.875,"hallu_top100":61,"inter_rater_agreement":0.6349206349206349,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2499,"model_cost_usd":0.02239},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6476337448559671,"score_ci_lo":0.580761316872428,"score_ci_hi":0.7134773662551441,"top100":65,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6476337448559671,"hallu_rate_mean":1.2962962962962963,"hallu_prevalence":1,"hallu_top100":67,"inter_rater_agreement":0.6959876543209877,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5233,"model_cost_usd":0.037784},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.13407738095238095,"score_ci_lo":0.08035714285714285,"score_ci_hi":0.1864583333333333,"top100":13,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.13407738095238095,"hallu_rate_mean":3.6666666666666665,"hallu_prevalence":1,"hallu_top100":8,"inter_rater_agreement":0.5901785714285714,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3775,"model_cost_usd":0.029168},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5125000000000001,"score_ci_lo":0.4,"score_ci_hi":0.6291666666666667,"top100":51,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5125000000000001,"hallu_rate_mean":3.2083333333333335,"hallu_prevalence":1,"hallu_top100":19,"inter_rater_agreement":0.5375,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6719,"model_cost_usd":0.049286}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7631794131794132,"score_ci_lo":0.7076799743466411,"score_ci_hi":0.8165263748597081,"top100":76,"n_cells":33,"hallu_rate_mean_across_domains":1.797979797979798,"hallu_top100_across_domains":55},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.4422077922077922,"score_ci_lo":0.3758136924803592,"score_ci_hi":0.506425364758698,"top100":44,"n_cells":33,"hallu_rate_mean_across_domains":1.0707070707070707,"hallu_top100_across_domains":73},{"model_id":"gpt-5","score_mean_across_domains":0.7092672759339426,"score_ci_lo":0.6237734487734488,"score_ci_hi":0.7794332210998878,"top100":71,"n_cells":33,"hallu_rate_mean_across_domains":1.5353535353535352,"hallu_top100_across_domains":61},{"model_id":"grok-4.3","score_mean_across_domains":0.42413419913419914,"score_ci_lo":0.34087702421035754,"score_ci_hi":0.5075156325156325,"top100":42,"n_cells":33,"hallu_rate_mean_across_domains":1.5606060606060606,"hallu_top100_across_domains":60},{"model_id":"mistral-large-2","score_mean_across_domains":0.4480118646785314,"score_ci_lo":0.3721540804874138,"score_ci_hi":0.5275773609106943,"top100":45,"n_cells":33,"hallu_rate_mean_across_domains":2.393939393939394,"hallu_top100_across_domains":40}],"cost":{"model_calls_usd":4.9984,"judge_calls_usd":23.6845,"total_usd":28.6829,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":522729,"tokens_out_total":110274,"cost_usd":16.111485},{"judge_id":"gpt-5","tokens_in_total":271717,"tokens_out_total":329456,"cost_usd":6.300425},{"judge_id":"mistral-large-2","tokens_in_total":288015,"tokens_out_total":116091,"cost_usd":1.272576}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":1043.02,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260516T200347-bc6991/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260516T200347-bc6991/progress.jsonl","cost_total_usd":28.6829}