{"run_id":"run-20260511T051320-09184c","started_at":1778476400,"summary":{"run_id":"run-20260511T051320-09184c","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778476400.017707,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.787092151675485,"score_ci_lo":0.7302414021164022,"score_ci_hi":0.8385967813051146,"top100":79,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.787092151675485,"hallu_rate_mean":0.9027777777777778,"hallu_prevalence":0.7916666666666666,"hallu_top100":77,"inter_rater_agreement":0.777281746031746,"n_judges":3,"tokens_in_total":18906,"tokens_out_total":21491,"model_cost_usd":1.895415},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8151577503429356,"score_ci_lo":0.7568587105624142,"score_ci_hi":0.8645404663923182,"top100":82,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.8151577503429356,"hallu_rate_mean":1.2839506172839505,"hallu_prevalence":1,"hallu_top100":67,"inter_rater_agreement":0.7906378600823045,"n_judges":3,"tokens_in_total":16515,"tokens_out_total":28159,"model_cost_usd":2.35965},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6930059523809523,"score_ci_lo":0.6289186507936508,"score_ci_hi":0.7629960317460318,"top100":69,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.6930059523809523,"hallu_rate_mean":2.430555555555556,"hallu_prevalence":1,"hallu_top100":39,"inter_rater_agreement":0.7891369047619047,"n_judges":3,"tokens_in_total":17337,"tokens_out_total":31374,"model_cost_usd":2.613105},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.6944444444444445,"score_ci_lo":0.6486111111111111,"score_ci_hi":0.7472222222222222,"top100":69,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.6944444444444445,"hallu_rate_mean":2.5416666666666665,"hallu_prevalence":1,"hallu_top100":36,"inter_rater_agreement":0.6208333333333333,"n_judges":3,"tokens_in_total":24912,"tokens_out_total":42919,"model_cost_usd":3.592605},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5525242504409171,"score_ci_lo":0.49090608465608465,"score_ci_hi":0.6101741622574955,"top100":55,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.5525242504409171,"hallu_rate_mean":0.3888888888888889,"hallu_prevalence":0.25,"hallu_top100":90,"inter_rater_agreement":0.6957671957671958,"n_judges":3,"tokens_in_total":10710,"tokens_out_total":5048,"model_cost_usd":0.063868},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5346364883401921,"score_ci_lo":0.4758230452674897,"score_ci_hi":0.5876200274348422,"top100":53,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.5346364883401921,"hallu_rate_mean":0.7037037037037037,"hallu_prevalence":0.6666666666666666,"hallu_top100":82,"inter_rater_agreement":0.705761316872428,"n_judges":3,"tokens_in_total":9426,"tokens_out_total":6319,"model_cost_usd":0.074972},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.30744047619047615,"score_ci_lo":0.23625992063492063,"score_ci_hi":0.37757936507936507,"top100":31,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.30744047619047615,"hallu_rate_mean":1.986111111111111,"hallu_prevalence":0.9583333333333334,"hallu_top100":50,"inter_rater_agreement":0.6513392857142858,"n_judges":3,"tokens_in_total":9510,"tokens_out_total":5510,"model_cost_usd":0.066988},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.4611111111111111,"score_ci_lo":0.37777777777777777,"score_ci_hi":0.5375,"top100":46,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.4611111111111111,"hallu_rate_mean":1.986111111111111,"hallu_prevalence":0.9166666666666666,"hallu_top100":50,"inter_rater_agreement":0.5666666666666667,"n_judges":3,"tokens_in_total":13071,"tokens_out_total":9428,"model_cost_usd":0.110619},{"model_id":"gpt-5","domain":"jura","score_mean":0.7090222663139331,"score_ci_lo":0.6612929894179894,"score_ci_hi":0.7560074955908288,"top100":71,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.7090222663139331,"hallu_rate_mean":0.9166666666666666,"hallu_prevalence":0.75,"hallu_top100":77,"inter_rater_agreement":0.7224702380952381,"n_judges":3,"tokens_in_total":9762,"tokens_out_total":41624,"model_cost_usd":0.67317},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8353909465020576,"score_ci_lo":0.8009259259259259,"score_ci_hi":0.8691700960219478,"top100":84,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.8353909465020576,"hallu_rate_mean":0.9629629629629629,"hallu_prevalence":1,"hallu_top100":75,"inter_rater_agreement":0.7654320987654322,"n_judges":3,"tokens_in_total":8985,"tokens_out_total":36044,"model_cost_usd":0.585585},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5083829365079365,"score_ci_lo":0.4134920634920635,"score_ci_hi":0.6040674603174604,"top100":51,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.5083829365079365,"hallu_rate_mean":2.7569444444444446,"hallu_prevalence":0.9583333333333334,"hallu_top100":31,"inter_rater_agreement":0.697767857142857,"n_judges":3,"tokens_in_total":9000,"tokens_out_total":67986,"model_cost_usd":1.06479},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7402777777777777,"score_ci_lo":0.6736111111111112,"score_ci_hi":0.7999999999999999,"top100":74,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.7402777777777777,"hallu_rate_mean":1.625,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.6291666666666667,"n_judges":3,"tokens_in_total":12636,"tokens_out_total":65510,"model_cost_usd":1.04583},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.46687610229276894,"score_ci_lo":0.41878858024691357,"score_ci_hi":0.5201719576719577,"top100":47,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.46687610229276894,"hallu_rate_mean":1.25,"hallu_prevalence":0.9166666666666666,"hallu_top100":68,"inter_rater_agreement":0.5991236772486772,"n_judges":3,"tokens_in_total":11094,"tokens_out_total":7064,"model_cost_usd":0.064572},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.666838134430727,"score_ci_lo":0.6246570644718793,"score_ci_hi":0.7028463648834019,"top100":67,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.666838134430727,"hallu_rate_mean":1.2098765432098764,"hallu_prevalence":1,"hallu_top100":69,"inter_rater_agreement":0.6599794238683127,"n_judges":3,"tokens_in_total":9579,"tokens_out_total":15923,"model_cost_usd":0.114696},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.11453373015873015,"score_ci_lo":0.07807539682539683,"score_ci_hi":0.1538690476190476,"top100":11,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.11453373015873015,"hallu_rate_mean":3.986111111111111,"hallu_prevalence":1,"hallu_top100":0,"inter_rater_agreement":0.6200892857142857,"n_judges":3,"tokens_in_total":9777,"tokens_out_total":11639,"model_cost_usd":0.089388},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.45,"score_ci_lo":0.36666666666666664,"score_ci_hi":0.5361111111111111,"top100":45,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.45,"hallu_rate_mean":3.138888888888889,"hallu_prevalence":1,"hallu_top100":21,"inter_rater_agreement":0.5083333333333333,"n_judges":3,"tokens_in_total":13458,"tokens_out_total":19760,"model_cost_usd":0.145476}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7494775800331356,"score_ci_lo":0.7191745497301053,"score_ci_hi":0.7798273742718187,"top100":75,"n_cells":99,"hallu_rate_mean_across_domains":1.7744107744107742,"hallu_top100_across_domains":55},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.46607076051520496,"score_ci_lo":0.42848591737480624,"score_ci_hi":0.5080567580567581,"top100":47,"n_cells":99,"hallu_rate_mean_across_domains":1.2491582491582491,"hallu_top100_across_domains":68},{"model_id":"gpt-5","score_mean_across_domains":0.7024237079792636,"score_ci_lo":0.6642790337234782,"score_ci_hi":0.7427048260381593,"top100":70,"n_cells":99,"hallu_rate_mean_across_domains":1.5471380471380471,"hallu_top100_across_domains":61},{"model_id":"mistral-large-2","score_mean_across_domains":0.4319036930148041,"score_ci_lo":0.3822363850141628,"score_ci_hi":0.48211479878146546,"top100":43,"n_cells":99,"hallu_rate_mean_across_domains":2.3602693602693603,"hallu_top100_across_domains":40}],"cost":{"model_calls_usd":14.5607,"judge_calls_usd":58.4744,"total_usd":73.0351,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1285328,"tokens_out_total":270036,"cost_usd":39.53262},{"judge_id":"gpt-5","tokens_in_total":672193,"tokens_out_total":827828,"cost_usd":15.778385},{"judge_id":"mistral-large-2","tokens_in_total":717573,"tokens_out_total":288042,"cost_usd":3.163398}],"extrapolation":{"scale_factor":12.12,"extrapolated_total_usd":885.27,"target_full_run_cells":1200,"observed_cells":99,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260511T051320-09184c/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260511T051320-09184c/progress.jsonl","cost_total_usd":73.0351}