{"run_id":"run-20260514T075711-8acb3a","started_at":1778750617,"summary":{"run_id":"run-20260514T075711-8acb3a","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778750617.5386188,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7711089065255732,"score_ci_lo":0.7137896825396824,"score_ci_hi":0.8219521604938271,"top100":77,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.7711089065255732,"hallu_rate_mean":1.1944444444444444,"hallu_prevalence":0.8333333333333334,"hallu_top100":70,"inter_rater_agreement":0.7319775132275131,"n_judges":3,"tokens_in_total":18906,"tokens_out_total":21311,"model_cost_usd":1.881915},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8167009602194787,"score_ci_lo":0.7590877914951989,"score_ci_hi":0.8679698216735253,"top100":82,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.8167009602194787,"hallu_rate_mean":1.2098765432098764,"hallu_prevalence":1,"hallu_top100":69,"inter_rater_agreement":0.8251028806584363,"n_judges":3,"tokens_in_total":16515,"tokens_out_total":28447,"model_cost_usd":2.38125},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6767361111111111,"score_ci_lo":0.6153769841269842,"score_ci_hi":0.7381944444444445,"top100":68,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.6767361111111111,"hallu_rate_mean":2.847222222222222,"hallu_prevalence":1,"hallu_top100":28,"inter_rater_agreement":0.7848214285714286,"n_judges":3,"tokens_in_total":17337,"tokens_out_total":32749,"model_cost_usd":2.71623},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7166666666666667,"score_ci_lo":0.6527777777777778,"score_ci_hi":0.7819444444444444,"top100":72,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.7166666666666667,"hallu_rate_mean":2.9166666666666665,"hallu_prevalence":1,"hallu_top100":27,"inter_rater_agreement":0.7125,"n_judges":3,"tokens_in_total":24912,"tokens_out_total":43234,"model_cost_usd":3.61623},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5096450617283951,"score_ci_lo":0.449211860670194,"score_ci_hi":0.5717868165784833,"top100":51,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.5096450617283951,"hallu_rate_mean":0.47222222222222227,"hallu_prevalence":0.4583333333333333,"hallu_top100":88,"inter_rater_agreement":0.7042824074074074,"n_judges":3,"tokens_in_total":10710,"tokens_out_total":4755,"model_cost_usd":0.060937},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5356652949245542,"score_ci_lo":0.49365569272976684,"score_ci_hi":0.5737311385459534,"top100":54,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.5356652949245542,"hallu_rate_mean":0.4444444444444444,"hallu_prevalence":0.7407407407407407,"hallu_top100":88,"inter_rater_agreement":0.713477366255144,"n_judges":3,"tokens_in_total":9426,"tokens_out_total":6261,"model_cost_usd":0.074393},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.29965277777777777,"score_ci_lo":0.22688492063492063,"score_ci_hi":0.37043650793650795,"top100":30,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.29965277777777777,"hallu_rate_mean":1.7916666666666667,"hallu_prevalence":0.875,"hallu_top100":55,"inter_rater_agreement":0.5885416666666666,"n_judges":3,"tokens_in_total":9510,"tokens_out_total":5508,"model_cost_usd":0.066967},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.4375,"score_ci_lo":0.36944444444444446,"score_ci_hi":0.5069444444444444,"top100":44,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.4375,"hallu_rate_mean":1.736111111111111,"hallu_prevalence":0.9166666666666666,"hallu_top100":56,"inter_rater_agreement":0.44166666666666665,"n_judges":3,"tokens_in_total":13071,"tokens_out_total":8890,"model_cost_usd":0.105239},{"model_id":"gpt-5","domain":"jura","score_mean":0.658344356261023,"score_ci_lo":0.6131227954144621,"score_ci_hi":0.7026014109347444,"top100":66,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.658344356261023,"hallu_rate_mean":0.9583333333333334,"hallu_prevalence":0.7083333333333334,"hallu_top100":76,"inter_rater_agreement":0.6424437830687831,"n_judges":3,"tokens_in_total":9762,"tokens_out_total":42535,"model_cost_usd":0.686835},{"model_id":"gpt-5","domain":"medizin","score_mean":0.7992112482853223,"score_ci_lo":0.7512002743484225,"score_ci_hi":0.8444787379972565,"top100":80,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.7992112482853223,"hallu_rate_mean":1,"hallu_prevalence":0.9629629629629629,"hallu_top100":75,"inter_rater_agreement":0.6815843621399177,"n_judges":3,"tokens_in_total":8985,"tokens_out_total":39284,"model_cost_usd":0.634185},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.529513888888889,"score_ci_lo":0.4390873015873016,"score_ci_hi":0.6192460317460318,"top100":53,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.529513888888889,"hallu_rate_mean":2.4027777777777777,"hallu_prevalence":0.9583333333333334,"hallu_top100":39,"inter_rater_agreement":0.7178571428571429,"n_judges":3,"tokens_in_total":9000,"tokens_out_total":66130,"model_cost_usd":1.03695},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.798611111111111,"score_ci_lo":0.7597222222222221,"score_ci_hi":0.8416666666666667,"top100":80,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.798611111111111,"hallu_rate_mean":1.625,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.7416666666666667,"n_judges":3,"tokens_in_total":12636,"tokens_out_total":68376,"model_cost_usd":1.08882},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.492146164021164,"score_ci_lo":0.4502039241622575,"score_ci_hi":0.53505291005291,"top100":49,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.492146164021164,"hallu_rate_mean":1.2083333333333333,"hallu_prevalence":0.875,"hallu_top100":69,"inter_rater_agreement":0.5850694444444444,"n_judges":3,"tokens_in_total":11094,"tokens_out_total":7678,"model_cost_usd":0.068256},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6747256515775034,"score_ci_lo":0.623113854595336,"score_ci_hi":0.7227366255144033,"top100":67,"n_cells":27,"closed_accuracy":null,"open_rubric_score":0.6747256515775034,"hallu_rate_mean":1.5185185185185186,"hallu_prevalence":0.9629629629629629,"hallu_top100":62,"inter_rater_agreement":0.6898148148148148,"n_judges":3,"tokens_in_total":9579,"tokens_out_total":16797,"model_cost_usd":0.11994},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.126140873015873,"score_ci_lo":0.0878968253968254,"score_ci_hi":0.1663690476190476,"top100":13,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.126140873015873,"hallu_rate_mean":3.6666666666666665,"hallu_prevalence":1,"hallu_top100":8,"inter_rater_agreement":0.6171130952380952,"n_judges":3,"tokens_in_total":9777,"tokens_out_total":11551,"model_cost_usd":0.08886},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5055555555555555,"score_ci_lo":0.4305555555555556,"score_ci_hi":0.5805555555555556,"top100":51,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.5055555555555555,"hallu_rate_mean":3.5416666666666665,"hallu_prevalence":1,"hallu_top100":11,"inter_rater_agreement":0.5750000000000001,"n_judges":3,"tokens_in_total":13458,"tokens_out_total":19690,"model_cost_usd":0.145056}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7474667308000641,"score_ci_lo":0.7156338517449629,"score_ci_hi":0.7768865907754797,"top100":75,"n_cells":99,"hallu_rate_mean_across_domains":2.016835016835017,"hallu_top100_across_domains":49},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.44834455667789,"score_ci_lo":0.41069424402757737,"score_ci_hi":0.4821441932553044,"top100":45,"n_cells":99,"hallu_rate_mean_across_domains":1.0909090909090908,"hallu_top100_across_domains":72},{"model_id":"gpt-5","score_mean_across_domains":0.6995350328683662,"score_ci_lo":0.6626463043129709,"score_ci_hi":0.7355846828069049,"top100":70,"n_cells":99,"hallu_rate_mean_across_domains":1.4814814814814814,"hallu_top100_across_domains":62},{"model_id":"mistral-large-2","score_mean_across_domains":0.4564627759072204,"score_ci_lo":0.40896531452087004,"score_ci_hi":0.505695847362514,"top100":46,"n_cells":99,"hallu_rate_mean_across_domains":2.4545454545454546,"hallu_top100_across_domains":38}],"cost":{"model_calls_usd":14.7721,"judge_calls_usd":58.7934,"total_usd":73.5655,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1297966,"tokens_out_total":271208,"cost_usd":39.81009},{"judge_id":"gpt-5","tokens_in_total":675015,"tokens_out_total":827557,"cost_usd":15.78843},{"judge_id":"mistral-large-2","tokens_in_total":720888,"tokens_out_total":292185,"cost_usd":3.194886}],"extrapolation":{"scale_factor":12.12,"extrapolated_total_usd":891.7,"target_full_run_cells":1200,"observed_cells":99,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260514T075711-8acb3a/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260514T075711-8acb3a/progress.jsonl","cost_total_usd":73.5655}