{"run_id":"run-20260510T190628-59fc01","started_at":1778439988,"summary":{"run_id":"run-20260510T190628-59fc01","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778439988.534802,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.769510582010582,"score_ci_lo":0.6883267195767195,"score_ci_hi":0.8478009259259259,"top100":77,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.769510582010582,"hallu_rate_mean":1.1666666666666667,"hallu_prevalence":1,"hallu_top100":70,"inter_rater_agreement":0.8306051587301587,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6632,"model_cost_usd":0.59193},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8251028806584362,"score_ci_lo":0.7283950617283951,"score_ci_hi":0.9027777777777778,"top100":83,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8251028806584362,"hallu_rate_mean":0.9259259259259258,"hallu_prevalence":1,"hallu_top100":76,"inter_rater_agreement":0.787037037037037,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9355,"model_cost_usd":0.7842},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7110119047619048,"score_ci_lo":0.6052083333333333,"score_ci_hi":0.8049107142857143,"top100":71,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7110119047619048,"hallu_rate_mean":2.2916666666666665,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.7955357142857142,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10774,"model_cost_usd":0.894735},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7083333333333333,"score_ci_lo":0.6333333333333333,"score_ci_hi":0.8041666666666666,"top100":71,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7083333333333333,"hallu_rate_mean":3.2083333333333335,"hallu_prevalence":1,"hallu_top100":19,"inter_rater_agreement":0.625,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":15026,"model_cost_usd":1.25151},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5147982804232805,"score_ci_lo":0.4305555555555556,"score_ci_hi":0.609375,"top100":51,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5147982804232805,"hallu_rate_mean":0.375,"hallu_prevalence":0.5,"hallu_top100":90,"inter_rater_agreement":0.7455357142857143,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1541,"model_cost_usd":0.019873},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5421810699588477,"score_ci_lo":0.47170781893004116,"score_ci_hi":0.6090534979423868,"top100":54,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5421810699588477,"hallu_rate_mean":0.5185185185185186,"hallu_prevalence":0.6666666666666666,"hallu_top100":87,"inter_rater_agreement":0.6867283950617283,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2164,"model_cost_usd":0.025567},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.3244047619047619,"score_ci_lo":0.18854166666666666,"score_ci_hi":0.4602678571428571,"top100":32,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.3244047619047619,"hallu_rate_mean":1.8333333333333333,"hallu_prevalence":1,"hallu_top100":54,"inter_rater_agreement":0.6102678571428571,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1898,"model_cost_usd":0.022943},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.38333333333333336,"score_ci_lo":0.2666666666666667,"score_ci_hi":0.525,"top100":38,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.38333333333333336,"hallu_rate_mean":1.7083333333333333,"hallu_prevalence":1,"hallu_top100":57,"inter_rater_agreement":0.4625,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":3091,"model_cost_usd":0.036356},{"model_id":"gpt-5","domain":"jura","score_mean":0.7313988095238095,"score_ci_lo":0.6614583333333334,"score_ci_hi":0.8015046296296297,"top100":73,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7313988095238095,"hallu_rate_mean":0.5,"hallu_prevalence":0.75,"hallu_top100":87,"inter_rater_agreement":0.683531746031746,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":15295,"model_cost_usd":0.245695},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8168724279835391,"score_ci_lo":0.7541152263374485,"score_ci_hi":0.8770576131687242,"top100":82,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8168724279835391,"hallu_rate_mean":0.925925925925926,"hallu_prevalence":0.8888888888888888,"hallu_top100":76,"inter_rater_agreement":0.7731481481481481,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":13818,"model_cost_usd":0.222245},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5526785714285715,"score_ci_lo":0.4395833333333333,"score_ci_hi":0.6803571428571429,"top100":55,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5526785714285715,"hallu_rate_mean":2.4583333333333335,"hallu_prevalence":1,"hallu_top100":38,"inter_rater_agreement":0.7508928571428571,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":21997,"model_cost_usd":0.344955},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7916666666666667,"score_ci_lo":0.7291666666666666,"score_ci_hi":0.8583333333333333,"top100":79,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7916666666666667,"hallu_rate_mean":1.7083333333333333,"hallu_prevalence":1,"hallu_top100":57,"inter_rater_agreement":0.75,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":22896,"model_cost_usd":0.3645},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.5400132275132274,"score_ci_lo":0.46593915343915343,"score_ci_hi":0.6348379629629629,"top100":54,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5400132275132274,"hallu_rate_mean":1,"hallu_prevalence":0.875,"hallu_top100":75,"inter_rater_agreement":0.621031746031746,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2662,"model_cost_usd":0.023368},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6358024691358025,"score_ci_lo":0.5231481481481483,"score_ci_hi":0.7587448559670782,"top100":64,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6358024691358025,"hallu_rate_mean":1.2962962962962963,"hallu_prevalence":1,"hallu_top100":67,"inter_rater_agreement":0.7407407407407408,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5222,"model_cost_usd":0.037718},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.08616071428571428,"score_ci_lo":0.04389880952380952,"score_ci_hi":0.134375,"top100":9,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.08616071428571428,"hallu_rate_mean":4.208333333333333,"hallu_prevalence":1,"hallu_top100":0,"inter_rater_agreement":0.5848214285714286,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3412,"model_cost_usd":0.02699},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5208333333333334,"score_ci_lo":0.3875,"score_ci_hi":0.6458333333333334,"top100":52,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5208333333333334,"hallu_rate_mean":3.3333333333333335,"hallu_prevalence":1,"hallu_top100":16,"inter_rater_agreement":0.575,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6749,"model_cost_usd":0.049466}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.755659772326439,"score_ci_lo":0.7058601891935224,"score_ci_hi":0.8049502966169633,"top100":76,"n_cells":33,"hallu_rate_mean_across_domains":1.8686868686868685,"hallu_top100_across_domains":53},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.4442400192400192,"score_ci_lo":0.3827441077441077,"score_ci_hi":0.5045654962321628,"top100":44,"n_cells":33,"hallu_rate_mean_across_domains":1.0909090909090908,"hallu_top100_across_domains":72},{"model_id":"gpt-5","score_mean_across_domains":0.7259940676607343,"score_ci_lo":0.6738576238576238,"score_ci_hi":0.7797498797498797,"top100":73,"n_cells":33,"hallu_rate_mean_across_domains":1.3838383838383839,"hallu_top100_across_domains":65},{"model_id":"mistral-large-2","score_mean_across_domains":0.4514630431297098,"score_ci_lo":0.3604697771364438,"score_ci_hi":0.5428210678210679,"top100":45,"n_cells":33,"hallu_rate_mean_across_domains":2.4242424242424243,"hallu_top100_across_domains":39}],"cost":{"model_calls_usd":4.9421,"judge_calls_usd":19.6421,"total_usd":24.5842,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":433707,"tokens_out_total":91184,"cost_usd":13.344405},{"judge_id":"gpt-5","tokens_in_total":225495,"tokens_out_total":273987,"cost_usd":5.23728},{"judge_id":"mistral-large-2","tokens_in_total":240865,"tokens_out_total":96454,"cost_usd":1.060454}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":893.97,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T190628-59fc01/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T190628-59fc01/progress.jsonl","cost_total_usd":24.5842}