{"run_id":"run-20260518T043700-b4a380","started_at":1779079020,"summary":{"run_id":"run-20260518T043700-b4a380","question_bank_version":"2026-05-17-v1","code_version":"0.1.0","started_at":1779079020.3819692,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8299433106575963,"score_ci_lo":0.7851587301587302,"score_ci_hi":0.8712169312169312,"top100":83,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8299433106575963,"hallu_rate_mean":0.9714285714285714,"hallu_prevalence":0.8571428571428571,"hallu_top100":75,"inter_rater_agreement":0.8325850340136054,"n_judges":3,"tokens_in_total":22950,"tokens_out_total":30010,"model_cost_usd":2.595},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8675595238095237,"score_ci_lo":0.8106150793650795,"score_ci_hi":0.9198412698412698,"top100":87,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8675595238095237,"hallu_rate_mean":1.2833333333333334,"hallu_prevalence":0.9,"hallu_top100":67,"inter_rater_agreement":0.827579365079365,"n_judges":3,"tokens_in_total":11495,"tokens_out_total":21175,"model_cost_usd":1.76055},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7289858906525574,"score_ci_lo":0.6820987654320988,"score_ci_hi":0.7801763668430335,"top100":73,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7289858906525574,"hallu_rate_mean":2.2444444444444445,"hallu_prevalence":1,"hallu_top100":43,"inter_rater_agreement":0.778994708994709,"n_judges":3,"tokens_in_total":9810,"tokens_out_total":19687,"model_cost_usd":1.623675},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7299559082892416,"score_ci_lo":0.6708201058201058,"score_ci_hi":0.7890828924162258,"top100":73,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7299559082892416,"hallu_rate_mean":2.433333333333333,"hallu_prevalence":1,"hallu_top100":39,"inter_rater_agreement":0.749074074074074,"n_judges":3,"tokens_in_total":22262,"tokens_out_total":42733,"model_cost_usd":3.538905},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6257445200302344,"score_ci_lo":0.5748979591836735,"score_ci_hi":0.6679667422524566,"top100":63,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6257445200302344,"hallu_rate_mean":0.3142857142857143,"hallu_prevalence":0.42857142857142855,"hallu_top100":92,"inter_rater_agreement":0.7224716553287982,"n_judges":3,"tokens_in_total":12898,"tokens_out_total":6713,"model_cost_usd":0.083252},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5635912698412698,"score_ci_lo":0.4848875661375661,"score_ci_hi":0.6375992063492063,"top100":56,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5635912698412698,"hallu_rate_mean":0.4166666666666667,"hallu_prevalence":0.45,"hallu_top100":89,"inter_rater_agreement":0.7627976190476191,"n_judges":3,"tokens_in_total":6530,"tokens_out_total":4140,"model_cost_usd":0.049562},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.39881834215167544,"score_ci_lo":0.31555555555555553,"score_ci_hi":0.49874779541446207,"top100":40,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.39881834215167544,"hallu_rate_mean":1.2888888888888888,"hallu_prevalence":0.8666666666666667,"hallu_top100":67,"inter_rater_agreement":0.5899206349206348,"n_judges":3,"tokens_in_total":5420,"tokens_out_total":3271,"model_cost_usd":0.039485},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.4811992945326279,"score_ci_lo":0.42050264550264554,"score_ci_hi":0.5485802469135803,"top100":48,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4811992945326279,"hallu_rate_mean":1.2111111111111112,"hallu_prevalence":0.7666666666666667,"hallu_top100":69,"inter_rater_agreement":0.5931481481481481,"n_judges":3,"tokens_in_total":11545,"tokens_out_total":8459,"model_cost_usd":0.099021},{"model_id":"gpt-5","domain":"jura","score_mean":0.7643046107331822,"score_ci_lo":0.7186772486772487,"score_ci_hi":0.81,"top100":76,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7643046107331822,"hallu_rate_mean":0.780952380952381,"hallu_prevalence":0.8571428571428571,"hallu_top100":80,"inter_rater_agreement":0.7217573696145124,"n_judges":3,"tokens_in_total":11850,"tokens_out_total":59222,"model_cost_usd":0.94758},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8181216931216932,"score_ci_lo":0.7300595238095238,"score_ci_hi":0.8833002645502646,"top100":82,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8181216931216932,"hallu_rate_mean":1.15,"hallu_prevalence":0.95,"hallu_top100":71,"inter_rater_agreement":0.8007936507936508,"n_judges":3,"tokens_in_total":6251,"tokens_out_total":25481,"model_cost_usd":0.41347},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5998324514991181,"score_ci_lo":0.5059435626102292,"score_ci_hi":0.6893386243386244,"top100":60,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5998324514991181,"hallu_rate_mean":1.8444444444444446,"hallu_prevalence":1,"hallu_top100":53,"inter_rater_agreement":0.6603174603174603,"n_judges":3,"tokens_in_total":5093,"tokens_out_total":39394,"model_cost_usd":0.616375},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7746516754850088,"score_ci_lo":0.7003042328042327,"score_ci_hi":0.8361992945326279,"top100":77,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7746516754850088,"hallu_rate_mean":1.422222222222222,"hallu_prevalence":1,"hallu_top100":64,"inter_rater_agreement":0.7370238095238094,"n_judges":3,"tokens_in_total":11129,"tokens_out_total":71121,"model_cost_usd":1.12246},{"model_id":"grok-4.3","domain":"jura","score_mean":0.5358578987150415,"score_ci_lo":0.48535525321239603,"score_ci_hi":0.583121693121693,"top100":54,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5358578987150415,"hallu_rate_mean":0.5904761904761905,"hallu_prevalence":0.6285714285714286,"hallu_top100":85,"inter_rater_agreement":0.675249433106576,"n_judges":3,"tokens_in_total":17561,"tokens_out_total":4853,"model_cost_usd":0.034084},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5100859788359788,"score_ci_lo":0.43392857142857144,"score_ci_hi":0.5815145502645502,"top100":51,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5100859788359788,"hallu_rate_mean":0.7166666666666666,"hallu_prevalence":0.75,"hallu_top100":82,"inter_rater_agreement":0.7271825396825398,"n_judges":3,"tokens_in_total":9055,"tokens_out_total":4593,"model_cost_usd":0.022801},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.22327160493827158,"score_ci_lo":0.14166666666666666,"score_ci_hi":0.31425925925925924,"top100":22,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.22327160493827158,"hallu_rate_mean":2.533333333333333,"hallu_prevalence":0.8666666666666667,"hallu_top100":36,"inter_rater_agreement":0.5851322751322752,"n_judges":3,"tokens_in_total":7535,"tokens_out_total":3642,"model_cost_usd":0.018524},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.44844356261022933,"score_ci_lo":0.3848809523809524,"score_ci_hi":0.5133730158730159,"top100":45,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.44844356261022933,"hallu_rate_mean":1.3111111111111111,"hallu_prevalence":0.9,"hallu_top100":67,"inter_rater_agreement":0.5273809523809524,"n_judges":3,"tokens_in_total":16294,"tokens_out_total":9032,"model_cost_usd":0.042947},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.502482993197279,"score_ci_lo":0.43160997732426304,"score_ci_hi":0.5721390778533636,"top100":50,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.502482993197279,"hallu_rate_mean":1.380952380952381,"hallu_prevalence":0.7714285714285715,"hallu_top100":65,"inter_rater_agreement":0.6805668934240363,"n_judges":3,"tokens_in_total":13332,"tokens_out_total":10465,"model_cost_usd":0.089454},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6356150793650793,"score_ci_lo":0.5607473544973545,"score_ci_hi":0.7092592592592593,"top100":64,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6356150793650793,"hallu_rate_mean":1.2166666666666666,"hallu_prevalence":0.9,"hallu_top100":69,"inter_rater_agreement":0.6698412698412699,"n_judges":3,"tokens_in_total":6669,"tokens_out_total":10451,"model_cost_usd":0.076044},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.23508818342151674,"score_ci_lo":0.15437389770723103,"score_ci_hi":0.3114197530864197,"top100":24,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.23508818342151674,"hallu_rate_mean":3.5777777777777775,"hallu_prevalence":1,"hallu_top100":10,"inter_rater_agreement":0.527010582010582,"n_judges":3,"tokens_in_total":5572,"tokens_out_total":7121,"model_cost_usd":0.05387},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5208818342151675,"score_ci_lo":0.45420194003527337,"score_ci_hi":0.5864638447971782,"top100":52,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5208818342151675,"hallu_rate_mean":2.344444444444444,"hallu_prevalence":0.9333333333333333,"hallu_top100":41,"inter_rater_agreement":0.5967328042328043,"n_judges":3,"tokens_in_total":11870,"tokens_out_total":19407,"model_cost_usd":0.140182}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7923267195767195,"score_ci_lo":0.7621124338624338,"score_ci_hi":0.8244021164021165,"top100":79,"n_cells":100,"hallu_rate_mean_across_domains":1.6633333333333333,"hallu_top100_across_domains":58},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5359113756613757,"score_ci_lo":0.4993346560846561,"score_ci_hi":0.5702777777777778,"top100":54,"n_cells":100,"hallu_rate_mean_across_domains":0.75,"hallu_top100_across_domains":81},{"model_id":"gpt-5","score_mean_across_domains":0.7535013227513229,"score_ci_lo":0.7158478835978836,"score_ci_hi":0.7894021164021164,"top100":75,"n_cells":100,"hallu_rate_mean_across_domains":1.2066666666666668,"hallu_top100_across_domains":69},{"model_id":"grok-4.3","score_mean_across_domains":0.45759126984126985,"score_ci_lo":0.4174126984126984,"score_ci_hi":0.4951957671957672,"top100":46,"n_cells":100,"hallu_rate_mean_across_domains":1.1233333333333333,"hallu_top100_across_domains":71},{"model_id":"mistral-large-2","score_mean_across_domains":0.4945198412698413,"score_ci_lo":0.45343650793650797,"score_ci_hi":0.5431097883597884,"top100":49,"n_cells":100,"hallu_rate_mean_across_domains":1.9666666666666666,"hallu_top100_across_domains":50}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8722,"oracle_top100":87,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.83,"best_single_model_top100":83},{"domain":"medizin","n_items":20,"oracle_mean":0.906,"oracle_top100":91,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8676,"best_single_model_top100":87},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7431,"oracle_top100":74,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.729,"best_single_model_top100":73},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8275,"oracle_top100":83,"best_single_model_id":"gpt-5","best_single_model_mean":0.7746,"best_single_model_top100":77}],"cost":{"model_calls_usd":13.3672,"judge_calls_usd":63.651,"total_usd":77.0183,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1359739,"tokens_out_total":302225,"cost_usd":43.06296},{"judge_id":"gpt-5","tokens_in_total":703400,"tokens_out_total":907073,"cost_usd":17.123095},{"judge_id":"mistral-large-2","tokens_in_total":754639,"tokens_out_total":325947,"cost_usd":3.46496}],"extrapolation":{"scale_factor":null,"extrapolated_total_usd":null,"note":"observed run too small or cost zero — extrapolation skipped"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260518T043700-b4a380/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260518T043700-b4a380/progress.jsonl","cost_total_usd":77.0183}