{"run_id":"run-20260529T043500-b4ffe0","started_at":1780029300,"summary":{"run_id":"run-20260529T043500-b4ffe0","question_bank_version":"2026-05-17-v1","code_version":"0.2.3","started_at":1780029300.9673,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8183673469387754,"score_ci_lo":0.7745181405895691,"score_ci_hi":0.8566468253968255,"top100":82,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8183673469387754,"hallu_rate_mean":0.9928571428571429,"hallu_prevalence":0.8571428571428571,"hallu_top100":75,"inter_rater_agreement":0.821281179138322,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":28417,"model_cost_usd":0.825175},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8424603174603174,"score_ci_lo":0.7729166666666667,"score_ci_hi":0.9042658730158729,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8424603174603174,"hallu_rate_mean":1.1375,"hallu_prevalence":1,"hallu_top100":71,"inter_rater_agreement":0.8143849206349205,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19778,"model_cost_usd":0.551925},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.71494708994709,"score_ci_lo":0.6575462962962962,"score_ci_hi":0.7691468253968254,"top100":71,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.71494708994709,"hallu_rate_mean":2.433333333333333,"hallu_prevalence":1,"hallu_top100":39,"inter_rater_agreement":0.7844708994708995,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":18689,"model_cost_usd":0.516275},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7691666666666667,"score_ci_lo":0.7231845238095238,"score_ci_hi":0.8129662698412699,"top100":77,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7691666666666667,"hallu_rate_mean":2.5083333333333333,"hallu_prevalence":1,"hallu_top100":37,"inter_rater_agreement":0.6808465608465608,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":42741,"model_cost_usd":1.179835},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7916383219954648,"score_ci_lo":0.7371428571428572,"score_ci_hi":0.8397335600907029,"top100":79,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7916383219954648,"hallu_rate_mean":0.5571428571428572,"hallu_prevalence":0.6571428571428571,"hallu_top100":86,"inter_rater_agreement":0.7560884353741497,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":32166,"model_cost_usd":0.918025},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.7856894841269841,"score_ci_lo":0.7219494047619047,"score_ci_hi":0.8436259920634921,"top100":79,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7856894841269841,"hallu_rate_mean":0.6875,"hallu_prevalence":0.8,"hallu_top100":82,"inter_rater_agreement":0.7241071428571428,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":20220,"model_cost_usd":0.562475},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6853108465608465,"score_ci_lo":0.6266534391534392,"score_ci_hi":0.7424999999999999,"top100":69,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6853108465608465,"hallu_rate_mean":1.4333333333333333,"hallu_prevalence":1,"hallu_top100":64,"inter_rater_agreement":0.7934656084656084,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":21143,"model_cost_usd":0.57725},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.6797255291005292,"score_ci_lo":0.625436507936508,"score_ci_hi":0.7360912698412698,"top100":68,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6797255291005292,"hallu_rate_mean":1.8583333333333334,"hallu_prevalence":0.9666666666666667,"hallu_top100":53,"inter_rater_agreement":0.618478835978836,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":47440,"model_cost_usd":1.29656},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6226785714285714,"score_ci_lo":0.5756944444444445,"score_ci_hi":0.6678571428571428,"top100":62,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6226785714285714,"hallu_rate_mean":0.2357142857142857,"hallu_prevalence":0.34285714285714286,"hallu_top100":94,"inter_rater_agreement":0.6804421768707483,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":6816,"model_cost_usd":0.084282},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.6053819444444445,"score_ci_lo":0.5230902777777777,"score_ci_hi":0.6744047619047618,"top100":61,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6053819444444445,"hallu_rate_mean":0.3625,"hallu_prevalence":0.6,"hallu_top100":90,"inter_rater_agreement":0.7343253968253969,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4239,"model_cost_usd":0.050553},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.37565476190476194,"score_ci_lo":0.27814814814814814,"score_ci_hi":0.47848544973544976,"top100":38,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.37565476190476194,"hallu_rate_mean":1.7833333333333334,"hallu_prevalence":0.8,"hallu_top100":55,"inter_rater_agreement":0.6001322751322751,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3331,"model_cost_usd":0.040085},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5188359788359789,"score_ci_lo":0.45306216931216936,"score_ci_hi":0.5845568783068783,"top100":52,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5188359788359789,"hallu_rate_mean":1.25,"hallu_prevalence":0.9,"hallu_top100":68,"inter_rater_agreement":0.6067328042328042,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":8736,"model_cost_usd":0.101791},{"model_id":"gpt-5","domain":"jura","score_mean":0.7342573696145125,"score_ci_lo":0.6584353741496598,"score_ci_hi":0.8002579365079365,"top100":73,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7342573696145125,"hallu_rate_mean":0.9642857142857143,"hallu_prevalence":0.8,"hallu_top100":75,"inter_rater_agreement":0.6997392290249433,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":58546,"model_cost_usd":0.93744},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8129464285714286,"score_ci_lo":0.7165922619047619,"score_ci_hi":0.8923859126984126,"top100":81,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8129464285714286,"hallu_rate_mean":0.9875,"hallu_prevalence":0.8,"hallu_top100":75,"inter_rater_agreement":0.7845238095238095,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":26476,"model_cost_usd":0.428395},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.627063492063492,"score_ci_lo":0.5178306878306879,"score_ci_hi":0.7256349206349206,"top100":63,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.627063492063492,"hallu_rate_mean":2.1666666666666665,"hallu_prevalence":1,"hallu_top100":45,"inter_rater_agreement":0.6874338624338624,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":39088,"model_cost_usd":0.611785},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.804021164021164,"score_ci_lo":0.733806216931217,"score_ci_hi":0.8553571428571428,"top100":80,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.804021164021164,"hallu_rate_mean":1.4333333333333333,"hallu_prevalence":0.9666666666666667,"hallu_top100":64,"inter_rater_agreement":0.753994708994709,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":68319,"model_cost_usd":1.08043},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4808673469387755,"score_ci_lo":0.4165816326530612,"score_ci_hi":0.5435799319727892,"top100":48,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4808673469387755,"hallu_rate_mean":0.42142857142857143,"hallu_prevalence":0.34285714285714286,"hallu_top100":89,"inter_rater_agreement":0.6693764172335601,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4097,"model_cost_usd":0.032194},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5534970238095238,"score_ci_lo":0.4662946428571429,"score_ci_hi":0.6437748015873016,"top100":55,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5534970238095238,"hallu_rate_mean":0.4125,"hallu_prevalence":0.6,"hallu_top100":89,"inter_rater_agreement":0.7482142857142857,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3791,"model_cost_usd":0.020796},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.2701190476190476,"score_ci_lo":0.17902777777777779,"score_ci_hi":0.3689616402116402,"top100":27,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2701190476190476,"hallu_rate_mean":2.433333333333333,"hallu_prevalence":1,"hallu_top100":39,"inter_rater_agreement":0.6008994708994708,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3323,"model_cost_usd":0.017726},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.49924603174603177,"score_ci_lo":0.4299503968253969,"score_ci_hi":0.5691964285714286,"top100":50,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.49924603174603177,"hallu_rate_mean":1.1333333333333333,"hallu_prevalence":0.8666666666666667,"hallu_top100":71,"inter_rater_agreement":0.5434656084656084,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":8252,"model_cost_usd":0.040998},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.46109410430839,"score_ci_lo":0.37722222222222224,"score_ci_hi":0.5386904761904763,"top100":46,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.46109410430839,"hallu_rate_mean":1.8857142857142857,"hallu_prevalence":0.7714285714285715,"hallu_top100":52,"inter_rater_agreement":0.6415986394557823,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10112,"model_cost_usd":0.087336},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6690724206349207,"score_ci_lo":0.6032242063492064,"score_ci_hi":0.7368303571428572,"top100":67,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6690724206349207,"hallu_rate_mean":1.175,"hallu_prevalence":0.95,"hallu_top100":70,"inter_rater_agreement":0.6966269841269841,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":9496,"model_cost_usd":0.070314},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.28347222222222224,"score_ci_lo":0.1940277777777778,"score_ci_hi":0.38939814814814816,"top100":28,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.28347222222222224,"hallu_rate_mean":3.4,"hallu_prevalence":0.9333333333333333,"hallu_top100":15,"inter_rater_agreement":0.4961111111111111,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":6752,"model_cost_usd":0.051656},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5880489417989417,"score_ci_lo":0.5211441798941799,"score_ci_hi":0.6459722222222222,"top100":59,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5880489417989417,"hallu_rate_mean":2.283333333333333,"hallu_prevalence":0.9666666666666667,"hallu_top100":42,"inter_rater_agreement":0.5813095238095237,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18921,"model_cost_usd":0.137266}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7929126984126984,"score_ci_lo":0.7656279761904762,"score_ci_hi":0.8171894841269841,"top100":79,"n_cells":100,"hallu_rate_mean_across_domains":1.6925,"hallu_top100_across_domains":57},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.7409255952380953,"score_ci_lo":0.7076865079365079,"score_ci_hi":0.7701051587301587,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.105,"hallu_top100_across_domains":72},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5510128968253968,"score_ci_lo":0.5135515873015872,"score_ci_hi":0.5874632936507936,"top100":55,"n_cells":100,"hallu_rate_mean_across_domains":0.7975,"hallu_top100_across_domains":80},{"model_id":"gpt-5","score_mean_across_domains":0.7548452380952381,"score_ci_lo":0.7117906746031746,"score_ci_hi":0.793969246031746,"top100":75,"n_cells":100,"hallu_rate_mean_across_domains":1.29,"hallu_top100_across_domains":67},{"model_id":"grok-4.3","score_mean_across_domains":0.4692946428571429,"score_ci_lo":0.430952380952381,"score_ci_hi":0.5098730158730159,"top100":47,"n_cells":100,"hallu_rate_mean_across_domains":0.935,"hallu_top100_across_domains":76},{"model_id":"mistral-large-2","score_mean_across_domains":0.5141329365079365,"score_ci_lo":0.4679573412698413,"score_ci_hi":0.5653492063492064,"top100":51,"n_cells":100,"hallu_rate_mean_across_domains":2.09,"hallu_top100_across_domains":47}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8744,"oracle_top100":87,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8184,"best_single_model_top100":82},{"domain":"medizin","n_items":20,"oracle_mean":0.9055,"oracle_top100":91,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8425,"best_single_model_top100":84},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7784,"oracle_top100":78,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7149,"best_single_model_top100":71},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8586,"oracle_top100":86,"best_single_model_id":"gpt-5","best_single_model_mean":0.804,"best_single_model_top100":80}],"cost":{"model_calls_usd":10.2206,"judge_calls_usd":62.663,"total_usd":72.8836,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1658970,"tokens_out_total":366075,"cost_usd":17.446725},{"judge_id":"claude-opus-4-8","tokens_in_total":1655970,"tokens_out_total":475868,"cost_usd":20.17655},{"judge_id":"gpt-5","tokens_in_total":858407,"tokens_out_total":1103097,"cost_usd":20.83849},{"judge_id":"mistral-large-2","tokens_in_total":920823,"tokens_out_total":393265,"cost_usd":4.201236}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":874.6,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260529T043500-b4ffe0/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260529T043500-b4ffe0/progress.jsonl","cost_total_usd":72.8836}