{"run_id":"run-20260512T105829-f0ac95","started_at":1778583509,"summary":{"run_id":"run-20260512T105829-f0ac95","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778583509.478232,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8529265873015873,"score_ci_lo":0.8095238095238095,"score_ci_hi":0.9039351851851852,"top100":85,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.8529265873015873,"hallu_rate_mean":0.8333333333333334,"hallu_prevalence":0.75,"hallu_top100":79,"inter_rater_agreement":0.8936011904761905,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6808,"model_cost_usd":0.60513},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8153292181069959,"score_ci_lo":0.7268518518518517,"score_ci_hi":0.8847736625514403,"top100":82,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8153292181069959,"hallu_rate_mean":1.5925925925925926,"hallu_prevalence":1,"hallu_top100":60,"inter_rater_agreement":0.816358024691358,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9267,"model_cost_usd":0.7776},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6678571428571429,"score_ci_lo":0.5677083333333334,"score_ci_hi":0.7534226190476191,"top100":67,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6678571428571429,"hallu_rate_mean":2.7916666666666665,"hallu_prevalence":1,"hallu_top100":30,"inter_rater_agreement":0.8321428571428572,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":11097,"model_cost_usd":0.91896},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7708333333333333,"score_ci_lo":0.7083333333333333,"score_ci_hi":0.8416666666666667,"top100":77,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7708333333333333,"hallu_rate_mean":2.9166666666666665,"hallu_prevalence":1,"hallu_top100":27,"inter_rater_agreement":0.7625,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":14712,"model_cost_usd":1.22796},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6010251322751323,"score_ci_lo":0.5264550264550264,"score_ci_hi":0.6788194444444444,"top100":60,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6010251322751323,"hallu_rate_mean":0.08333333333333333,"hallu_prevalence":0.25,"hallu_top100":97,"inter_rater_agreement":0.7160218253968254,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1746,"model_cost_usd":0.021922},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5689300411522633,"score_ci_lo":0.44804526748971196,"score_ci_hi":0.6790123456790123,"top100":57,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5689300411522633,"hallu_rate_mean":0.48148148148148145,"hallu_prevalence":0.6666666666666666,"hallu_top100":87,"inter_rater_agreement":0.6265432098765432,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2122,"model_cost_usd":0.025148},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.3087797619047619,"score_ci_lo":0.18333333333333332,"score_ci_hi":0.45863095238095236,"top100":31,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.3087797619047619,"hallu_rate_mean":1.9583333333333333,"hallu_prevalence":0.875,"hallu_top100":51,"inter_rater_agreement":0.7071428571428572,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1778,"model_cost_usd":0.021743},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.45416666666666666,"score_ci_lo":0.3208333333333333,"score_ci_hi":0.5791666666666667,"top100":45,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.45416666666666666,"hallu_rate_mean":1.8333333333333333,"hallu_prevalence":1,"hallu_top100":54,"inter_rater_agreement":0.6,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":2803,"model_cost_usd":0.033476},{"model_id":"gpt-5","domain":"jura","score_mean":0.7209821428571429,"score_ci_lo":0.627066798941799,"score_ci_hi":0.8090277777777778,"top100":72,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7209821428571429,"hallu_rate_mean":0.625,"hallu_prevalence":0.75,"hallu_top100":84,"inter_rater_agreement":0.7155257936507936,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":14345,"model_cost_usd":0.231445},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8204732510288066,"score_ci_lo":0.7469135802469136,"score_ci_hi":0.8873456790123457,"top100":82,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8204732510288066,"hallu_rate_mean":1.0370370370370372,"hallu_prevalence":1,"hallu_top100":74,"inter_rater_agreement":0.7623456790123456,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":12321,"model_cost_usd":0.19979},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.4980654761904762,"score_ci_lo":0.3541666666666667,"score_ci_hi":0.6285714285714286,"top100":50,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4980654761904762,"hallu_rate_mean":2.5,"hallu_prevalence":1,"hallu_top100":37,"inter_rater_agreement":0.6486607142857143,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":22062,"model_cost_usd":0.34593},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.8416666666666667,"score_ci_lo":0.7875000000000001,"score_ci_hi":0.9,"top100":84,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.8416666666666667,"hallu_rate_mean":1.625,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.8125,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":22838,"model_cost_usd":0.36363},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.5567956349206349,"score_ci_lo":0.49661044973544977,"score_ci_hi":0.6192129629629629,"top100":56,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5567956349206349,"hallu_rate_mean":0.7083333333333333,"hallu_prevalence":0.75,"hallu_top100":82,"inter_rater_agreement":0.5932539682539683,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2714,"model_cost_usd":0.02368},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.669238683127572,"score_ci_lo":0.5910493827160495,"score_ci_hi":0.7448559670781892,"top100":67,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.669238683127572,"hallu_rate_mean":1.3333333333333333,"hallu_prevalence":1,"hallu_top100":66,"inter_rater_agreement":0.7469135802469136,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5993,"model_cost_usd":0.042344},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.16770833333333332,"score_ci_lo":0.08541666666666667,"score_ci_hi":0.253125,"top100":17,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.16770833333333332,"hallu_rate_mean":3.875,"hallu_prevalence":1,"hallu_top100":3,"inter_rater_agreement":0.5361607142857143,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3544,"model_cost_usd":0.027782},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5333333333333333,"score_ci_lo":0.37916666666666665,"score_ci_hi":0.6416666666666666,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5333333333333333,"hallu_rate_mean":3.4583333333333335,"hallu_prevalence":1,"hallu_top100":13,"inter_rater_agreement":0.5625,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":7286,"model_cost_usd":0.052688}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7779060445727113,"score_ci_lo":0.7334135000801667,"score_ci_hi":0.8215247715247715,"top100":78,"n_cells":33,"hallu_rate_mean_across_domains":2.0202020202020203,"hallu_top100_across_domains":49},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.4858225108225109,"score_ci_lo":0.4151715568382235,"score_ci_hi":0.5535473785473786,"top100":49,"n_cells":33,"hallu_rate_mean_across_domains":1.0707070707070707,"hallu_top100_across_domains":73},{"model_id":"gpt-5","score_mean_across_domains":0.723332531665865,"score_ci_lo":0.6525813692480359,"score_ci_hi":0.788997113997114,"top100":72,"n_cells":33,"hallu_rate_mean_across_domains":1.4343434343434345,"hallu_top100_across_domains":64},{"model_id":"mistral-large-2","score_mean_across_domains":0.48744989578322917,"score_ci_lo":0.4098124098124098,"score_ci_hi":0.5671156004489338,"top100":49,"n_cells":33,"hallu_rate_mean_across_domains":2.313131313131313,"hallu_top100_across_domains":42}],"cost":{"model_calls_usd":4.9192,"judge_calls_usd":19.6639,"total_usd":24.5831,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":434383,"tokens_out_total":90722,"cost_usd":13.319895},{"judge_id":"gpt-5","tokens_in_total":225895,"tokens_out_total":277575,"cost_usd":5.2931},{"judge_id":"mistral-large-2","tokens_in_total":241199,"tokens_out_total":94746,"cost_usd":1.050874}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":893.93,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T105829-f0ac95/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T105829-f0ac95/progress.jsonl","cost_total_usd":24.5831}