{"run_id":"run-20260517T131406-f96128","started_at":1779023646,"summary":{"run_id":"run-20260517T131406-f96128","question_bank_version":"2026-05-17-v1","code_version":"0.1.0","started_at":1779023646.1912482,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8029969765684052,"score_ci_lo":0.73994708994709,"score_ci_hi":0.8589758125472411,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8029969765684052,"hallu_rate_mean":0.9523809523809524,"hallu_prevalence":0.7714285714285715,"hallu_top100":76,"inter_rater_agreement":0.7865532879818594,"n_judges":3,"tokens_in_total":22950,"tokens_out_total":29213,"model_cost_usd":2.535225},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8404100529100529,"score_ci_lo":0.7746693121693122,"score_ci_hi":0.892989417989418,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8404100529100529,"hallu_rate_mean":1.2166666666666666,"hallu_prevalence":1,"hallu_top100":69,"inter_rater_agreement":0.8384920634920634,"n_judges":3,"tokens_in_total":11495,"tokens_out_total":19533,"model_cost_usd":1.6374},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7591446208112875,"score_ci_lo":0.6816666666666666,"score_ci_hi":0.8396913580246913,"top100":76,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7591446208112875,"hallu_rate_mean":2.288888888888889,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.7225132275132274,"n_judges":3,"tokens_in_total":9810,"tokens_out_total":19584,"model_cost_usd":1.61595},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7454497354497355,"score_ci_lo":0.69,"score_ci_hi":0.7971693121693122,"top100":75,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7454497354497355,"hallu_rate_mean":2.388888888888889,"hallu_prevalence":1,"hallu_top100":40,"inter_rater_agreement":0.7155952380952381,"n_judges":3,"tokens_in_total":22262,"tokens_out_total":40894,"model_cost_usd":3.40098},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6155366591080876,"score_ci_lo":0.5647430083144369,"score_ci_hi":0.6547089947089947,"top100":62,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6155366591080876,"hallu_rate_mean":0.35238095238095235,"hallu_prevalence":0.2857142857142857,"hallu_top100":91,"inter_rater_agreement":0.7012471655328798,"n_judges":3,"tokens_in_total":12898,"tokens_out_total":6940,"model_cost_usd":0.085523},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5556878306878307,"score_ci_lo":0.47397486772486774,"score_ci_hi":0.6264880952380952,"top100":56,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5556878306878307,"hallu_rate_mean":0.4833333333333333,"hallu_prevalence":0.45,"hallu_top100":87,"inter_rater_agreement":0.7653769841269841,"n_judges":3,"tokens_in_total":6530,"tokens_out_total":4081,"model_cost_usd":0.048973},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.34163139329805997,"score_ci_lo":0.252989417989418,"score_ci_hi":0.43653439153439155,"top100":34,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.34163139329805997,"hallu_rate_mean":1.7333333333333334,"hallu_prevalence":0.8666666666666667,"hallu_top100":56,"inter_rater_agreement":0.623915343915344,"n_judges":3,"tokens_in_total":5420,"tokens_out_total":3142,"model_cost_usd":0.038195},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5235185185185185,"score_ci_lo":0.45940476190476187,"score_ci_hi":0.5873677248677249,"top100":52,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5235185185185185,"hallu_rate_mean":1.1888888888888889,"hallu_prevalence":0.7333333333333333,"hallu_top100":70,"inter_rater_agreement":0.6251587301587301,"n_judges":3,"tokens_in_total":11545,"tokens_out_total":8683,"model_cost_usd":0.101261},{"model_id":"gpt-5","domain":"jura","score_mean":0.757339380196523,"score_ci_lo":0.6932350718065003,"score_ci_hi":0.8100907029478458,"top100":76,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.757339380196523,"hallu_rate_mean":0.9619047619047618,"hallu_prevalence":0.8,"hallu_top100":75,"inter_rater_agreement":0.7065192743764173,"n_judges":3,"tokens_in_total":11850,"tokens_out_total":58707,"model_cost_usd":0.939855},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8058862433862434,"score_ci_lo":0.7323412698412699,"score_ci_hi":0.866005291005291,"top100":81,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8058862433862434,"hallu_rate_mean":1.2,"hallu_prevalence":0.95,"hallu_top100":70,"inter_rater_agreement":0.7954365079365079,"n_judges":3,"tokens_in_total":6251,"tokens_out_total":25437,"model_cost_usd":0.41281},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.593615520282187,"score_ci_lo":0.4727777777777778,"score_ci_hi":0.7059347442680776,"top100":59,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.593615520282187,"hallu_rate_mean":2.066666666666667,"hallu_prevalence":0.9333333333333333,"hallu_top100":48,"inter_rater_agreement":0.6547354497354497,"n_judges":3,"tokens_in_total":5093,"tokens_out_total":39282,"model_cost_usd":0.614695},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7773059964726631,"score_ci_lo":0.7237918871252205,"score_ci_hi":0.8273280423280424,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7773059964726631,"hallu_rate_mean":1.3555555555555554,"hallu_prevalence":0.9666666666666667,"hallu_top100":66,"inter_rater_agreement":0.6980026455026456,"n_judges":3,"tokens_in_total":11129,"tokens_out_total":70341,"model_cost_usd":1.11076},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4939380196523054,"score_ci_lo":0.4409183673469388,"score_ci_hi":0.5409486016628874,"top100":49,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4939380196523054,"hallu_rate_mean":0.6761904761904761,"hallu_prevalence":0.6,"hallu_top100":83,"inter_rater_agreement":0.6619160997732426,"n_judges":3,"tokens_in_total":17561,"tokens_out_total":4913,"model_cost_usd":0.034234},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5230820105820106,"score_ci_lo":0.4573743386243386,"score_ci_hi":0.592228835978836,"top100":52,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5230820105820106,"hallu_rate_mean":0.21666666666666665,"hallu_prevalence":0.35,"hallu_top100":94,"inter_rater_agreement":0.7475198412698413,"n_judges":3,"tokens_in_total":9055,"tokens_out_total":4155,"model_cost_usd":0.021706},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.25993827160493826,"score_ci_lo":0.1725925925925926,"score_ci_hi":0.34648148148148145,"top100":26,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.25993827160493826,"hallu_rate_mean":2.0444444444444447,"hallu_prevalence":0.8,"hallu_top100":48,"inter_rater_agreement":0.6111375661375662,"n_judges":3,"tokens_in_total":7535,"tokens_out_total":3544,"model_cost_usd":0.018279},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.4706922398589065,"score_ci_lo":0.4037345679012346,"score_ci_hi":0.5372619047619048,"top100":47,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4706922398589065,"hallu_rate_mean":1.2333333333333334,"hallu_prevalence":0.7,"hallu_top100":69,"inter_rater_agreement":0.5930555555555556,"n_judges":3,"tokens_in_total":16294,"tokens_out_total":8766,"model_cost_usd":0.042283},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4658238851095993,"score_ci_lo":0.39281179138322,"score_ci_hi":0.5395049130763416,"top100":47,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4658238851095993,"hallu_rate_mean":1.7142857142857142,"hallu_prevalence":0.8285714285714286,"hallu_top100":57,"inter_rater_agreement":0.6544444444444444,"n_judges":3,"tokens_in_total":13332,"tokens_out_total":10573,"model_cost_usd":0.090102},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6353835978835979,"score_ci_lo":0.5768518518518518,"score_ci_hi":0.694675925925926,"top100":64,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6353835978835979,"hallu_rate_mean":1.15,"hallu_prevalence":0.95,"hallu_top100":71,"inter_rater_agreement":0.6766865079365079,"n_judges":3,"tokens_in_total":6669,"tokens_out_total":10057,"model_cost_usd":0.07368},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.2898148148148148,"score_ci_lo":0.2002469135802469,"score_ci_hi":0.37296296296296294,"top100":29,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2898148148148148,"hallu_rate_mean":2.9777777777777774,"hallu_prevalence":1,"hallu_top100":25,"inter_rater_agreement":0.5285185185185185,"n_judges":3,"tokens_in_total":5572,"tokens_out_total":7118,"model_cost_usd":0.053852},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5743650793650794,"score_ci_lo":0.5086375661375662,"score_ci_hi":0.6389814814814815,"top100":57,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5743650793650794,"hallu_rate_mean":2.2555555555555555,"hallu_prevalence":0.9333333333333333,"hallu_top100":43,"inter_rater_agreement":0.5851058201058201,"n_judges":3,"tokens_in_total":11870,"tokens_out_total":18809,"model_cost_usd":0.136594}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7866375661375662,"score_ci_lo":0.752744708994709,"score_ci_hi":0.8155886243386243,"top100":79,"n_cells":100,"hallu_rate_mean_across_domains":1.6366666666666665,"hallu_top100_across_domains":59},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5348756613756613,"score_ci_lo":0.4958227513227513,"score_ci_hi":0.5723716931216931,"top100":53,"n_cells":100,"hallu_rate_mean_across_domains":0.8366666666666667,"hallu_top100_across_domains":79},{"model_id":"gpt-5","score_mean_across_domains":0.7484801587301587,"score_ci_lo":0.7128267195767195,"score_ci_hi":0.7794708994708995,"top100":75,"n_cells":100,"hallu_rate_mean_across_domains":1.2933333333333334,"hallu_top100_across_domains":67},{"model_id":"grok-4.3","score_mean_across_domains":0.45769312169312165,"score_ci_lo":0.42063492063492064,"score_ci_hi":0.49793915343915346,"top100":46,"n_cells":100,"hallu_rate_mean_across_domains":0.9566666666666667,"hallu_top100_across_domains":76},{"model_id":"mistral-large-2","score_mean_across_domains":0.5058968253968255,"score_ci_lo":0.46338095238095234,"score_ci_hi":0.5471574074074074,"top100":51,"n_cells":100,"hallu_rate_mean_across_domains":1.9533333333333334,"hallu_top100_across_domains":51}],"cost":{"model_calls_usd":13.0124,"judge_calls_usd":63.9672,"total_usd":76.9795,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1351058,"tokens_out_total":305990,"cost_usd":43.21512},{"judge_id":"gpt-5","tokens_in_total":699050,"tokens_out_total":919455,"cost_usd":17.287075},{"judge_id":"mistral-large-2","tokens_in_total":750094,"tokens_out_total":327466,"cost_usd":3.464984}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":923.75,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260517T131406-f96128/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260517T131406-f96128/progress.jsonl","cost_total_usd":76.9795}