{"run_id":"run-20260520T074328-4a549c","started_at":1779263008,"summary":{"run_id":"run-20260520T074328-4a549c","question_bank_version":"2026-05-17-v1","code_version":"0.2.3","started_at":1779263008.994366,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7715419501133787,"score_ci_lo":0.7023696145124717,"score_ci_hi":0.8258616780045351,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7715419501133787,"hallu_rate_mean":1.2476190476190476,"hallu_prevalence":0.8571428571428571,"hallu_top100":68,"inter_rater_agreement":0.8287755102040816,"n_judges":3,"tokens_in_total":22950,"tokens_out_total":30007,"model_cost_usd":2.594775},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.804596560846561,"score_ci_lo":0.7324074074074074,"score_ci_hi":0.8682208994708995,"top100":80,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.804596560846561,"hallu_rate_mean":1.4666666666666666,"hallu_prevalence":0.95,"hallu_top100":63,"inter_rater_agreement":0.821329365079365,"n_judges":3,"tokens_in_total":11495,"tokens_out_total":19548,"model_cost_usd":1.638525},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7874074074074073,"score_ci_lo":0.7245061728395061,"score_ci_hi":0.8566666666666667,"top100":79,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7874074074074073,"hallu_rate_mean":2.488888888888889,"hallu_prevalence":1,"hallu_top100":37,"inter_rater_agreement":0.7694444444444444,"n_judges":3,"tokens_in_total":9810,"tokens_out_total":19458,"model_cost_usd":1.6065},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7161331569664903,"score_ci_lo":0.6717328042328041,"score_ci_hi":0.7586904761904762,"top100":72,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7161331569664903,"hallu_rate_mean":2.2555555555555555,"hallu_prevalence":0.9666666666666667,"hallu_top100":43,"inter_rater_agreement":0.6577777777777778,"n_judges":3,"tokens_in_total":22262,"tokens_out_total":42170,"model_cost_usd":3.49668},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6163454270597127,"score_ci_lo":0.5623809523809524,"score_ci_hi":0.6641156462585034,"top100":62,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6163454270597127,"hallu_rate_mean":0.2476190476190476,"hallu_prevalence":0.2857142857142857,"hallu_top100":93,"inter_rater_agreement":0.7246938775510204,"n_judges":3,"tokens_in_total":12898,"tokens_out_total":6637,"model_cost_usd":0.082492},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5638558201058201,"score_ci_lo":0.4745039682539683,"score_ci_hi":0.6432539682539683,"top100":56,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5638558201058201,"hallu_rate_mean":0.4833333333333333,"hallu_prevalence":0.5,"hallu_top100":87,"inter_rater_agreement":0.7294642857142857,"n_judges":3,"tokens_in_total":6530,"tokens_out_total":4117,"model_cost_usd":0.049333},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.2998148148148148,"score_ci_lo":0.20555555555555555,"score_ci_hi":0.4003703703703704,"top100":30,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2998148148148148,"hallu_rate_mean":1.8,"hallu_prevalence":0.8666666666666667,"hallu_top100":55,"inter_rater_agreement":0.5894708994708995,"n_judges":3,"tokens_in_total":5420,"tokens_out_total":3228,"model_cost_usd":0.039055},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.49960317460317455,"score_ci_lo":0.43417989417989417,"score_ci_hi":0.5577513227513227,"top100":50,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.49960317460317455,"hallu_rate_mean":1.0888888888888888,"hallu_prevalence":0.8333333333333334,"hallu_top100":72,"inter_rater_agreement":0.6373941798941799,"n_judges":3,"tokens_in_total":11545,"tokens_out_total":8669,"model_cost_usd":0.101121},{"model_id":"gpt-5","domain":"jura","score_mean":0.7684278155706727,"score_ci_lo":0.7071768707482994,"score_ci_hi":0.8227399848828421,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7684278155706727,"hallu_rate_mean":0.9523809523809524,"hallu_prevalence":0.8285714285714286,"hallu_top100":76,"inter_rater_agreement":0.7293310657596372,"n_judges":3,"tokens_in_total":11850,"tokens_out_total":55180,"model_cost_usd":0.88695},{"model_id":"gpt-5","domain":"medizin","score_mean":0.7944113756613757,"score_ci_lo":0.6939814814814815,"score_ci_hi":0.877281746031746,"top100":79,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7944113756613757,"hallu_rate_mean":1.2166666666666666,"hallu_prevalence":0.85,"hallu_top100":69,"inter_rater_agreement":0.8054563492063492,"n_judges":3,"tokens_in_total":6251,"tokens_out_total":26662,"model_cost_usd":0.431185},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5415873015873015,"score_ci_lo":0.3972222222222222,"score_ci_hi":0.6724338624338624,"top100":54,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5415873015873015,"hallu_rate_mean":2.511111111111111,"hallu_prevalence":0.9333333333333333,"hallu_top100":37,"inter_rater_agreement":0.6476190476190475,"n_judges":3,"tokens_in_total":5093,"tokens_out_total":37389,"model_cost_usd":0.5863},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7795590828924162,"score_ci_lo":0.7232539682539681,"score_ci_hi":0.8323148148148148,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7795590828924162,"hallu_rate_mean":1.222222222222222,"hallu_prevalence":0.9333333333333333,"hallu_top100":69,"inter_rater_agreement":0.7348412698412699,"n_judges":3,"tokens_in_total":11129,"tokens_out_total":68873,"model_cost_usd":1.08874},{"model_id":"grok-4.3","domain":"jura","score_mean":0.5049584278155707,"score_ci_lo":0.45195389266817837,"score_ci_hi":0.5584240362811791,"top100":50,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5049584278155707,"hallu_rate_mean":0.4476190476190476,"hallu_prevalence":0.5142857142857142,"hallu_top100":88,"inter_rater_agreement":0.6791383219954649,"n_judges":3,"tokens_in_total":17561,"tokens_out_total":4588,"model_cost_usd":0.033421},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5507275132275132,"score_ci_lo":0.4810185185185185,"score_ci_hi":0.6102182539682539,"top100":55,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5507275132275132,"hallu_rate_mean":0.4333333333333333,"hallu_prevalence":0.7,"hallu_top100":89,"inter_rater_agreement":0.7046626984126985,"n_judges":3,"tokens_in_total":9055,"tokens_out_total":4209,"model_cost_usd":0.021841},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.3022222222222222,"score_ci_lo":0.22253086419753085,"score_ci_hi":0.37481481481481477,"top100":30,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3022222222222222,"hallu_rate_mean":2.3333333333333335,"hallu_prevalence":0.9333333333333333,"hallu_top100":41,"inter_rater_agreement":0.557116402116402,"n_judges":3,"tokens_in_total":7535,"tokens_out_total":3826,"model_cost_usd":0.018984},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.4889153439153439,"score_ci_lo":0.4239153439153439,"score_ci_hi":0.5573412698412699,"top100":49,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4889153439153439,"hallu_rate_mean":1.3777777777777778,"hallu_prevalence":0.7666666666666667,"hallu_top100":65,"inter_rater_agreement":0.6025661375661375,"n_judges":3,"tokens_in_total":16294,"tokens_out_total":9330,"model_cost_usd":0.043692},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.46179516250944824,"score_ci_lo":0.38221844293272866,"score_ci_hi":0.5438057445200303,"top100":46,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.46179516250944824,"hallu_rate_mean":1.838095238095238,"hallu_prevalence":0.8285714285714286,"hallu_top100":54,"inter_rater_agreement":0.6200566893424035,"n_judges":3,"tokens_in_total":13332,"tokens_out_total":10505,"model_cost_usd":0.089694},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.665542328042328,"score_ci_lo":0.5935185185185186,"score_ci_hi":0.7363756613756614,"top100":67,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.665542328042328,"hallu_rate_mean":1.25,"hallu_prevalence":0.95,"hallu_top100":68,"inter_rater_agreement":0.6841269841269841,"n_judges":3,"tokens_in_total":6669,"tokens_out_total":9767,"model_cost_usd":0.07194},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.27212522045855375,"score_ci_lo":0.18771604938271605,"score_ci_hi":0.36681657848324517,"top100":27,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.27212522045855375,"hallu_rate_mean":3.0444444444444443,"hallu_prevalence":1,"hallu_top100":23,"inter_rater_agreement":0.5525661375661375,"n_judges":3,"tokens_in_total":5572,"tokens_out_total":7231,"model_cost_usd":0.05453},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5244003527336861,"score_ci_lo":0.45534391534391533,"score_ci_hi":0.5868783068783068,"top100":52,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5244003527336861,"hallu_rate_mean":2.211111111111111,"hallu_prevalence":0.9333333333333333,"hallu_top100":44,"inter_rater_agreement":0.5604232804232805,"n_judges":3,"tokens_in_total":11870,"tokens_out_total":19528,"model_cost_usd":0.140908}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7639100529100529,"score_ci_lo":0.7330859788359788,"score_ci_hi":0.7934417989417989,"top100":76,"n_cells":100,"hallu_rate_mean_across_domains":1.78,"hallu_top100_across_domains":55},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5233452380952381,"score_ci_lo":0.4796441798941799,"score_ci_hi":0.5654325396825396,"top100":52,"n_cells":100,"hallu_rate_mean_across_domains":0.78,"hallu_top100_across_domains":80},{"model_id":"gpt-5","score_mean_across_domains":0.7429378306878307,"score_ci_lo":0.696660052910053,"score_ci_hi":0.7812619047619047,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.32,"hallu_top100_across_domains":67},{"model_id":"grok-4.3","score_mean_across_domains":0.47888888888888886,"score_ci_lo":0.4416931216931217,"score_ci_hi":0.5146375661375662,"top100":48,"n_cells":100,"hallu_rate_mean_across_domains":1.0066666666666668,"hallu_top100_across_domains":74},{"model_id":"mistral-large-2","score_mean_across_domains":0.49287566137566136,"score_ci_lo":0.44629100529100524,"score_ci_hi":0.5367857142857143,"top100":49,"n_cells":100,"hallu_rate_mean_across_domains":2.0133333333333336,"hallu_top100_across_domains":49}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8345,"oracle_top100":83,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7715,"best_single_model_top100":77},{"domain":"medizin","n_items":20,"oracle_mean":0.8801,"oracle_top100":88,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8046,"best_single_model_top100":80},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7919,"oracle_top100":79,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7874,"best_single_model_top100":79},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8178,"oracle_top100":82,"best_single_model_id":"gpt-5","best_single_model_mean":0.7795,"best_single_model_top100":78}],"cost":{"model_calls_usd":13.0767,"judge_calls_usd":63.6659,"total_usd":76.7426,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1356429,"tokens_out_total":301783,"cost_usd":42.98016},{"judge_id":"gpt-5","tokens_in_total":701828,"tokens_out_total":913670,"cost_usd":17.21419},{"judge_id":"mistral-large-2","tokens_in_total":752943,"tokens_out_total":327614,"cost_usd":3.47157}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":920.91,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260520T074328-4a549c/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260520T074328-4a549c/progress.jsonl","cost_total_usd":76.7426}