{"run_id":"run-20260620T073912-d2125c","started_at":1781941152,"summary":{"run_id":"run-20260620T073912-d2125c","question_bank_version":"2026-05-17-v1","code_version":"0.2.6","started_at":1781941152.6677947,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8187868480725623,"score_ci_lo":0.768687641723356,"score_ci_hi":0.8631746031746032,"top100":82,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8187868480725623,"hallu_rate_mean":1.0142857142857142,"hallu_prevalence":0.8285714285714286,"hallu_top100":74,"inter_rater_agreement":0.8206462585034013,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":29211,"model_cost_usd":0.845025},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8326140873015874,"score_ci_lo":0.753968253968254,"score_ci_hi":0.899454365079365,"top100":83,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8326140873015874,"hallu_rate_mean":1.2625,"hallu_prevalence":1,"hallu_top100":68,"inter_rater_agreement":0.8770833333333334,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19967,"model_cost_usd":0.55665},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.782989417989418,"score_ci_lo":0.7265079365079364,"score_ci_hi":0.8382936507936508,"top100":78,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.782989417989418,"hallu_rate_mean":2.066666666666667,"hallu_prevalence":1,"hallu_top100":48,"inter_rater_agreement":0.7967724867724867,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":18795,"model_cost_usd":0.518925},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7697619047619048,"score_ci_lo":0.7228075396825397,"score_ci_hi":0.813343253968254,"top100":77,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7697619047619048,"hallu_rate_mean":2.3,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.6853571428571429,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":40649,"model_cost_usd":1.127535},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7989965986394557,"score_ci_lo":0.7435374149659864,"score_ci_hi":0.8485969387755102,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7989965986394557,"hallu_rate_mean":0.6642857142857143,"hallu_prevalence":0.7142857142857143,"hallu_top100":83,"inter_rater_agreement":0.7638548752834466,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":33030,"model_cost_usd":0.939625},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.8132936507936508,"score_ci_lo":0.7455109126984126,"score_ci_hi":0.874826388888889,"top100":81,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8132936507936508,"hallu_rate_mean":0.6125,"hallu_prevalence":0.9,"hallu_top100":84,"inter_rater_agreement":0.7009920634920636,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":20640,"model_cost_usd":0.572975},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6540608465608465,"score_ci_lo":0.5922685185185185,"score_ci_hi":0.7173478835978836,"top100":65,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6540608465608465,"hallu_rate_mean":1.55,"hallu_prevalence":1,"hallu_top100":61,"inter_rater_agreement":0.7425132275132275,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":20207,"model_cost_usd":0.55385},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.673452380952381,"score_ci_lo":0.6209722222222223,"score_ci_hi":0.7313492063492063,"top100":67,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.673452380952381,"hallu_rate_mean":1.6333333333333333,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.6284656084656085,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":45544,"model_cost_usd":1.24916},{"model_id":"gemini-3-1-pro","domain":"jura","score_mean":0.5777210884353742,"score_ci_lo":0.5146485260770975,"score_ci_hi":0.6395436507936508,"top100":58,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5777210884353742,"hallu_rate_mean":0.7285714285714285,"hallu_prevalence":0.6,"hallu_top100":81,"inter_rater_agreement":0.7058843537414966,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":6418,"model_cost_usd":0.102812},{"model_id":"gemini-3-1-pro","domain":"medizin","score_mean":0.6298115079365079,"score_ci_lo":0.5653025793650793,"score_ci_hi":0.6967261904761906,"top100":63,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6298115079365079,"hallu_rate_mean":0.4375,"hallu_prevalence":0.6,"hallu_top100":89,"inter_rater_agreement":0.698015873015873,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4366,"model_cost_usd":0.065452},{"model_id":"gemini-3-1-pro","domain":"steuerrecht","score_mean":0.43837301587301586,"score_ci_lo":0.364484126984127,"score_ci_hi":0.5132275132275133,"top100":44,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.43837301587301586,"hallu_rate_mean":1.1333333333333333,"hallu_prevalence":0.7333333333333333,"hallu_top100":71,"inter_rater_agreement":0.5568783068783069,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3649,"model_cost_usd":0.054628},{"model_id":"gemini-3-1-pro","domain":"wirtschaftsrecht","score_mean":0.4866236772486773,"score_ci_lo":0.40855489417989416,"score_ci_hi":0.5632738095238096,"top100":49,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4866236772486773,"hallu_rate_mean":1.5666666666666667,"hallu_prevalence":0.8333333333333334,"hallu_top100":60,"inter_rater_agreement":0.5586375661375661,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":9256,"model_cost_usd":0.134162},{"model_id":"gpt-5","domain":"jura","score_mean":0.7728769841269841,"score_ci_lo":0.7315873015873016,"score_ci_hi":0.8118849206349207,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7728769841269841,"hallu_rate_mean":0.6857142857142857,"hallu_prevalence":0.7714285714285715,"hallu_top100":82,"inter_rater_agreement":0.714047619047619,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":61230,"model_cost_usd":0.9777},{"model_id":"gpt-5","domain":"medizin","score_mean":0.7848214285714286,"score_ci_lo":0.7029017857142856,"score_ci_hi":0.85625,"top100":78,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7848214285714286,"hallu_rate_mean":1.0125,"hallu_prevalence":1,"hallu_top100":74,"inter_rater_agreement":0.7012896825396825,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":27608,"model_cost_usd":0.445375},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.6672949735449735,"score_ci_lo":0.5409722222222223,"score_ci_hi":0.7893981481481481,"top100":67,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6672949735449735,"hallu_rate_mean":2.033333333333333,"hallu_prevalence":1,"hallu_top100":49,"inter_rater_agreement":0.7352910052910053,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":38521,"model_cost_usd":0.60328},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.760949074074074,"score_ci_lo":0.7040376984126984,"score_ci_hi":0.8143948412698413,"top100":76,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.760949074074074,"hallu_rate_mean":1.4583333333333333,"hallu_prevalence":1,"hallu_top100":63,"inter_rater_agreement":0.6454629629629629,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":70770,"model_cost_usd":1.117195},{"model_id":"gpt-5.5-pro","domain":"jura","score_mean":0.7718424036281178,"score_ci_lo":0.7038407029478458,"score_ci_hi":0.8269019274376418,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7718424036281178,"hallu_rate_mean":0.6071428571428571,"hallu_prevalence":0.7142857142857143,"hallu_top100":84,"inter_rater_agreement":0.77921768707483,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":282436,"model_cost_usd":8.53233},{"model_id":"gpt-5.5-pro","domain":"medizin","score_mean":0.8684027777777779,"score_ci_lo":0.8255208333333334,"score_ci_hi":0.9086805555555555,"top100":87,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8684027777777779,"hallu_rate_mean":0.6875,"hallu_prevalence":0.8,"hallu_top100":82,"inter_rater_agreement":0.8083333333333333,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":73504,"model_cost_usd":2.236375},{"model_id":"gpt-5.5-pro","domain":"steuerrecht","score_mean":0.678954081632653,"score_ci_lo":0.5973214285714287,"score_ci_hi":0.7609835600907029,"top100":68,"n_cells":14,"closed_accuracy":null,"open_rubric_score":0.678954081632653,"hallu_rate_mean":1.5535714285714286,"hallu_prevalence":1,"hallu_top100":61,"inter_rater_agreement":0.6833616780045351,"n_judges":4,"tokens_in_total":28401,"tokens_out_total":193938,"model_cost_usd":5.960145},{"model_id":"gpt-5.5-pro","domain":"wirtschaftsrecht","score_mean":0.7791236772486773,"score_ci_lo":0.7062698412698414,"score_ci_hi":0.8384325396825397,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7791236772486773,"hallu_rate_mean":0.9666666666666667,"hallu_prevalence":0.9666666666666667,"hallu_top100":75,"inter_rater_agreement":0.8152777777777778,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":277925,"model_cost_usd":8.393395},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4921570294784581,"score_ci_lo":0.43255385487528347,"score_ci_hi":0.5449206349206349,"top100":49,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4921570294784581,"hallu_rate_mean":0.6142857142857143,"hallu_prevalence":0.6,"hallu_top100":84,"inter_rater_agreement":0.6689002267573697,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4322,"model_cost_usd":0.032756},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.597296626984127,"score_ci_lo":0.503125,"score_ci_hi":0.6840029761904762,"top100":60,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.597296626984127,"hallu_rate_mean":0.5625,"hallu_prevalence":0.75,"hallu_top100":85,"inter_rater_agreement":0.7838293650793651,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3937,"model_cost_usd":0.021161},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.2823015873015873,"score_ci_lo":0.20921296296296296,"score_ci_hi":0.3694179894179894,"top100":28,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2823015873015873,"hallu_rate_mean":1.85,"hallu_prevalence":0.9333333333333333,"hallu_top100":53,"inter_rater_agreement":0.5420899470899471,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3246,"model_cost_usd":0.017534},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.4837070105820106,"score_ci_lo":0.41172619047619047,"score_ci_hi":0.5559523809523809,"top100":48,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4837070105820106,"hallu_rate_mean":1.4583333333333333,"hallu_prevalence":0.8666666666666667,"hallu_top100":63,"inter_rater_agreement":0.5719179894179894,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":8590,"model_cost_usd":0.041843},{"model_id":"kimi-k2.6","domain":"jura","score_mean":0.6273781179138322,"score_ci_lo":0.5678713151927438,"score_ci_hi":0.6855328798185941,"top100":63,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6273781179138322,"hallu_rate_mean":1.1214285714285714,"hallu_prevalence":0.7714285714285715,"hallu_top100":71,"inter_rater_agreement":0.6606462585034013,"n_judges":4,"tokens_in_total":15188,"tokens_out_total":264318,"model_cost_usd":1.071701},{"model_id":"kimi-k2.6","domain":"medizin","score_mean":0.6998263888888889,"score_ci_lo":0.6293402777777778,"score_ci_hi":0.7684027777777778,"top100":70,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6998263888888889,"hallu_rate_mean":0.85,"hallu_prevalence":0.9,"hallu_top100":78,"inter_rater_agreement":0.6714285714285715,"n_judges":4,"tokens_in_total":7536,"tokens_out_total":101872,"model_cost_usd":0.414647},{"model_id":"kimi-k2.6","domain":"steuerrecht","score_mean":0.30443594104308386,"score_ci_lo":0.18263180272108845,"score_ci_hi":0.42177579365079365,"top100":30,"n_cells":14,"closed_accuracy":null,"open_rubric_score":0.30443594104308386,"hallu_rate_mean":3.857142857142857,"hallu_prevalence":1,"hallu_top100":3,"inter_rater_agreement":0.45535714285714285,"n_judges":4,"tokens_in_total":6001,"tokens_out_total":133956,"model_cost_usd":0.541525},{"model_id":"kimi-k2.6","domain":"wirtschaftsrecht","score_mean":0.6114153439153439,"score_ci_lo":0.5463458994708995,"score_ci_hi":0.6696924603174603,"top100":61,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6114153439153439,"hallu_rate_mean":2.683333333333333,"hallu_prevalence":1,"hallu_top100":32,"inter_rater_agreement":0.5188624338624339,"n_judges":4,"tokens_in_total":14354,"tokens_out_total":258488,"model_cost_usd":1.047588},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4477692743764172,"score_ci_lo":0.37017006802721086,"score_ci_hi":0.5244246031746032,"top100":45,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4477692743764172,"hallu_rate_mean":1.8428571428571427,"hallu_prevalence":0.8,"hallu_top100":53,"inter_rater_agreement":0.6206462585034014,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10566,"model_cost_usd":0.09006},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6882688492063492,"score_ci_lo":0.6264880952380952,"score_ci_hi":0.7536706349206349,"top100":69,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6882688492063492,"hallu_rate_mean":0.95,"hallu_prevalence":0.85,"hallu_top100":76,"inter_rater_agreement":0.7213293650793651,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10064,"model_cost_usd":0.073722},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.2731679894179894,"score_ci_lo":0.17763227513227514,"score_ci_hi":0.3710185185185185,"top100":27,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2731679894179894,"hallu_rate_mean":3.4833333333333334,"hallu_prevalence":1,"hallu_top100":12,"inter_rater_agreement":0.5236507936507936,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":7247,"model_cost_usd":0.054626},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5703538359788359,"score_ci_lo":0.502146164021164,"score_ci_hi":0.6318882275132275,"top100":57,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5703538359788359,"hallu_rate_mean":2.05,"hallu_prevalence":0.9333333333333333,"hallu_top100":48,"inter_rater_agreement":0.5318783068783068,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18039,"model_cost_usd":0.131974}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.8014751984126984,"score_ci_lo":0.7736180555555555,"score_ci_hi":0.8264474206349206,"top100":80,"n_cells":100,"hallu_rate_mean_across_domains":1.6075,"hallu_top100_across_domains":59},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.742452380952381,"score_ci_lo":0.7080327380952381,"score_ci_hi":0.7736617063492064,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.0775,"hallu_top100_across_domains":73},{"model_id":"gemini-3-1-pro","score_mean_across_domains":0.5399077380952381,"score_ci_lo":0.496500992063492,"score_ci_hi":0.5822519841269841,"top100":54,"n_cells":100,"hallu_rate_mean_across_domains":0.9825,"hallu_top100_across_domains":75},{"model_id":"gpt-5","score_mean_across_domains":0.7558501984126984,"score_ci_lo":0.7202876984126984,"score_ci_hi":0.7906805555555556,"top100":76,"n_cells":100,"hallu_rate_mean_across_domains":1.185,"hallu_top100_across_domains":70},{"model_id":"gpt-5.5-pro","score_mean_across_domains":0.7804202741702742,"score_ci_lo":0.7468263989097322,"score_ci_hi":0.8151184463684463,"top100":78,"n_cells":99,"hallu_rate_mean_across_domains":0.8661616161616161,"hallu_top100_across_domains":78},{"model_id":"grok-4.3","score_mean_across_domains":0.47917162698412696,"score_ci_lo":0.4344712301587302,"score_ci_hi":0.5208422619047619,"top100":48,"n_cells":100,"hallu_rate_mean_across_domains":1.0425,"hallu_top100_across_domains":73},{"model_id":"kimi-k2.6","score_mean_across_domains":0.5915083373416706,"score_ci_lo":0.5434684143017476,"score_ci_hi":0.6350609267275934,"top100":59,"n_cells":99,"hallu_rate_mean_across_domains":1.9267676767676767,"hallu_top100_across_domains":51},{"model_id":"mistral-large-2","score_mean_across_domains":0.5064543650793651,"score_ci_lo":0.4583363095238095,"score_ci_hi":0.5565109126984127,"top100":51,"n_cells":100,"hallu_rate_mean_across_domains":1.9725,"hallu_top100_across_domains":50}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.889,"oracle_top100":89,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8188,"best_single_model_top100":82},{"domain":"medizin","n_items":20,"oracle_mean":0.9262,"oracle_top100":93,"best_single_model_id":"gpt-5.5-pro","best_single_model_mean":0.8684,"best_single_model_top100":87},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.8438,"oracle_top100":84,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.783,"best_single_model_top100":78},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8535,"oracle_top100":85,"best_single_model_id":"gpt-5.5-pro","best_single_model_mean":0.7791,"best_single_model_top100":78}],"cost":{"model_calls_usd":38.5257,"judge_calls_usd":84.0794,"total_usd":122.6051,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":2236186,"tokens_out_total":490561,"cost_usd":23.444955},{"judge_id":"claude-opus-4-8","tokens_in_total":2232196,"tokens_out_total":611102,"cost_usd":26.43853},{"judge_id":"gpt-5","tokens_in_total":1153431,"tokens_out_total":1516243,"cost_usd":28.5108},{"judge_id":"mistral-large-2","tokens_in_total":1237678,"tokens_out_total":534955,"cost_usd":5.685086}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":1471.26,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260620T073912-d2125c/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260620T073912-d2125c/progress.jsonl","cost_total_usd":122.6051}