{"run_id":"run-20260512T081552-d5fb45","started_at":1778573752,"summary":{"run_id":"run-20260512T081552-d5fb45","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778573752.1881754,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8035714285714286,"score_ci_lo":0.7483465608465608,"score_ci_hi":0.8547453703703703,"top100":80,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.8035714285714286,"hallu_rate_mean":1,"hallu_prevalence":0.875,"hallu_top100":75,"inter_rater_agreement":0.8658234126984127,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":7164,"model_cost_usd":0.63183},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8518518518518519,"score_ci_lo":0.7530864197530864,"score_ci_hi":0.9279835390946501,"top100":85,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8518518518518519,"hallu_rate_mean":0.8888888888888888,"hallu_prevalence":1,"hallu_top100":77,"inter_rater_agreement":0.8379629629629629,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9554,"model_cost_usd":0.799125},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7092261904761905,"score_ci_lo":0.6375,"score_ci_hi":0.7882440476190475,"top100":71,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7092261904761905,"hallu_rate_mean":2.625,"hallu_prevalence":1,"hallu_top100":34,"inter_rater_agreement":0.8236607142857143,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10139,"model_cost_usd":0.84711},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7083333333333334,"score_ci_lo":0.625,"score_ci_hi":0.8083333333333333,"top100":71,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7083333333333334,"hallu_rate_mean":3.4166666666666665,"hallu_prevalence":1,"hallu_top100":14,"inter_rater_agreement":0.6,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":14493,"model_cost_usd":1.211535},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5053736772486772,"score_ci_lo":0.41294642857142855,"score_ci_hi":0.5839120370370371,"top100":51,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5053736772486772,"hallu_rate_mean":0.375,"hallu_prevalence":0.375,"hallu_top100":90,"inter_rater_agreement":0.6396329365079365,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1521,"model_cost_usd":0.019672},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5416666666666666,"score_ci_lo":0.44495884773662553,"score_ci_hi":0.6342592592592592,"top100":54,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5416666666666666,"hallu_rate_mean":0.5925925925925926,"hallu_prevalence":0.6666666666666666,"hallu_top100":85,"inter_rater_agreement":0.6975308641975309,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2191,"model_cost_usd":0.025837},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.325,"score_ci_lo":0.2,"score_ci_hi":0.46458333333333335,"top100":32,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.325,"hallu_rate_mean":1.9166666666666667,"hallu_prevalence":0.875,"hallu_top100":52,"inter_rater_agreement":0.6526785714285714,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1968,"model_cost_usd":0.023643},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5291666666666667,"score_ci_lo":0.375,"score_ci_hi":0.6458333333333334,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5291666666666667,"hallu_rate_mean":1.75,"hallu_prevalence":1,"hallu_top100":56,"inter_rater_agreement":0.55,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":2844,"model_cost_usd":0.033886},{"model_id":"gpt-5","domain":"jura","score_mean":0.640625,"score_ci_lo":0.43865740740740744,"score_ci_hi":0.7743055555555556,"top100":64,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.640625,"hallu_rate_mean":1.3333333333333333,"hallu_prevalence":0.75,"hallu_top100":66,"inter_rater_agreement":0.7003968253968254,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":15776,"model_cost_usd":0.25291},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8549382716049383,"score_ci_lo":0.7839506172839505,"score_ci_hi":0.91820987654321,"top100":85,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8549382716049383,"hallu_rate_mean":1.074074074074074,"hallu_prevalence":0.8888888888888888,"hallu_top100":73,"inter_rater_agreement":0.8009259259259259,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":12793,"model_cost_usd":0.20687},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.6287202380952381,"score_ci_lo":0.49375,"score_ci_hi":0.7559523809523809,"top100":63,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6287202380952381,"hallu_rate_mean":2.3333333333333335,"hallu_prevalence":1,"hallu_top100":41,"inter_rater_agreement":0.7696428571428572,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":23368,"model_cost_usd":0.36552},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7666666666666666,"score_ci_lo":0.6791666666666667,"score_ci_hi":0.8416666666666667,"top100":77,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7666666666666666,"hallu_rate_mean":1.75,"hallu_prevalence":1,"hallu_top100":56,"inter_rater_agreement":0.7125,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":21816,"model_cost_usd":0.3483},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.5641534391534392,"score_ci_lo":0.482473544973545,"score_ci_hi":0.6498015873015873,"top100":56,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5641534391534392,"hallu_rate_mean":0.9166666666666666,"hallu_prevalence":0.75,"hallu_top100":77,"inter_rater_agreement":0.6014384920634921,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2483,"model_cost_usd":0.022294},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6985596707818931,"score_ci_lo":0.6378600823045267,"score_ci_hi":0.7664609053497942,"top100":70,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6985596707818931,"hallu_rate_mean":1.4074074074074074,"hallu_prevalence":1,"hallu_top100":64,"inter_rater_agreement":0.8024691358024691,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5710,"model_cost_usd":0.040646},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.12053571428571427,"score_ci_lo":0.06994047619047619,"score_ci_hi":0.16666666666666666,"top100":12,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.12053571428571427,"hallu_rate_mean":3.5416666666666665,"hallu_prevalence":1,"hallu_top100":11,"inter_rater_agreement":0.5955357142857143,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3663,"model_cost_usd":0.028496},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.4916666666666667,"score_ci_lo":0.3875,"score_ci_hi":0.5916666666666667,"top100":49,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4916666666666667,"hallu_rate_mean":2.875,"hallu_prevalence":1,"hallu_top100":28,"inter_rater_agreement":0.5125,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":5971,"model_cost_usd":0.044798}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7707792207792208,"score_ci_lo":0.7244829244829245,"score_ci_hi":0.814774731441398,"top100":77,"n_cells":33,"hallu_rate_mean_across_domains":1.9494949494949494,"hallu_top100_across_domains":51},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.477312810646144,"score_ci_lo":0.4165664582331249,"score_ci_hi":0.5394340227673561,"top100":48,"n_cells":33,"hallu_rate_mean_across_domains":1.1414141414141414,"hallu_top100_across_domains":71},{"model_id":"gpt-5","score_mean_across_domains":0.7267436267436267,"score_ci_lo":0.65331088664422,"score_ci_hi":0.7858385441718775,"top100":73,"n_cells":33,"hallu_rate_mean_across_domains":1.606060606060606,"hallu_top100_across_domains":59},{"model_id":"mistral-large-2","score_mean_across_domains":0.475693442360109,"score_ci_lo":0.392368125701459,"score_ci_hi":0.559359467692801,"top100":48,"n_cells":33,"hallu_rate_mean_across_domains":2.1616161616161613,"hallu_top100_across_domains":45}],"cost":{"model_calls_usd":4.9025,"judge_calls_usd":19.6734,"total_usd":24.5758,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":432874,"tokens_out_total":90967,"cost_usd":13.315635},{"judge_id":"gpt-5","tokens_in_total":225181,"tokens_out_total":278528,"cost_usd":5.303825},{"judge_id":"mistral-large-2","tokens_in_total":240272,"tokens_out_total":95558,"cost_usd":1.053892}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":893.67,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T081552-d5fb45/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260512T081552-d5fb45/progress.jsonl","cost_total_usd":24.5758}