{"run_id":"run-20260607T181746-23cbb6","started_at":1780856266,"summary":{"run_id":"run-20260607T181746-23cbb6","question_bank_version":"2026-05-17-v1","code_version":"0.2.3","started_at":1780856266.8635888,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8402069160997733,"score_ci_lo":0.8043934240362812,"score_ci_hi":0.8713293650793651,"top100":84,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8402069160997733,"hallu_rate_mean":0.9214285714285714,"hallu_prevalence":0.8571428571428571,"hallu_top100":76,"inter_rater_agreement":0.8037074829931973,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":29411,"model_cost_usd":0.850025},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8562996031746032,"score_ci_lo":0.7973958333333333,"score_ci_hi":0.9074404761904763,"top100":86,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8562996031746032,"hallu_rate_mean":1.125,"hallu_prevalence":0.95,"hallu_top100":71,"inter_rater_agreement":0.8016865079365079,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19204,"model_cost_usd":0.537575},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7666137566137566,"score_ci_lo":0.7087433862433862,"score_ci_hi":0.8151719576719576,"top100":77,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7666137566137566,"hallu_rate_mean":2.65,"hallu_prevalence":1,"hallu_top100":33,"inter_rater_agreement":0.7824338624338625,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":18885,"model_cost_usd":0.521175},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7575396825396825,"score_ci_lo":0.6992658730158731,"score_ci_hi":0.8116964285714285,"top100":76,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7575396825396825,"hallu_rate_mean":2.3,"hallu_prevalence":0.9666666666666667,"hallu_top100":42,"inter_rater_agreement":0.7024470899470899,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":43091,"model_cost_usd":1.188585},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7874773242630386,"score_ci_lo":0.7247590702947845,"score_ci_hi":0.8390079365079365,"top100":79,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7874773242630386,"hallu_rate_mean":0.7642857142857142,"hallu_prevalence":0.6285714285714286,"hallu_top100":80,"inter_rater_agreement":0.7556009070294785,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":31547,"model_cost_usd":0.90255},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.8020585317460318,"score_ci_lo":0.7371279761904762,"score_ci_hi":0.8656001984126984,"top100":80,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8020585317460318,"hallu_rate_mean":0.5875,"hallu_prevalence":0.85,"hallu_top100":85,"inter_rater_agreement":0.7486111111111111,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":20602,"model_cost_usd":0.572025},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6921230158730158,"score_ci_lo":0.6325,"score_ci_hi":0.7554431216931217,"top100":69,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6921230158730158,"hallu_rate_mean":1.4,"hallu_prevalence":0.9333333333333333,"hallu_top100":65,"inter_rater_agreement":0.7337301587301588,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":20261,"model_cost_usd":0.5552},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.6653472222222222,"score_ci_lo":0.610297619047619,"score_ci_hi":0.7206349206349206,"top100":67,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6653472222222222,"hallu_rate_mean":1.6833333333333333,"hallu_prevalence":0.9333333333333333,"hallu_top100":57,"inter_rater_agreement":0.6464153439153438,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":45610,"model_cost_usd":1.25081},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6249631519274377,"score_ci_lo":0.5830357142857143,"score_ci_hi":0.6656150793650794,"top100":62,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6249631519274377,"hallu_rate_mean":0.2785714285714286,"hallu_prevalence":0.4,"hallu_top100":93,"inter_rater_agreement":0.6892630385487528,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":7181,"model_cost_usd":0.087932},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5690972222222223,"score_ci_lo":0.4870535714285714,"score_ci_hi":0.6440972222222222,"top100":57,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5690972222222223,"hallu_rate_mean":0.575,"hallu_prevalence":0.65,"hallu_top100":85,"inter_rater_agreement":0.7051587301587301,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4228,"model_cost_usd":0.050443},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.3485515873015873,"score_ci_lo":0.25517195767195766,"score_ci_hi":0.43671296296296297,"top100":35,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3485515873015873,"hallu_rate_mean":1.7,"hallu_prevalence":0.8,"hallu_top100":57,"inter_rater_agreement":0.6185714285714285,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3356,"model_cost_usd":0.040335},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5050099206349207,"score_ci_lo":0.4373015873015873,"score_ci_hi":0.5724007936507937,"top100":51,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5050099206349207,"hallu_rate_mean":1.1333333333333333,"hallu_prevalence":0.7333333333333333,"hallu_top100":71,"inter_rater_agreement":0.5888624338624339,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":8573,"model_cost_usd":0.100161},{"model_id":"gpt-5","domain":"jura","score_mean":0.7433390022675737,"score_ci_lo":0.6775907029478457,"score_ci_hi":0.8020493197278912,"top100":74,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7433390022675737,"hallu_rate_mean":0.8928571428571429,"hallu_prevalence":0.8,"hallu_top100":77,"inter_rater_agreement":0.748968253968254,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":58920,"model_cost_usd":0.94305},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8237847222222221,"score_ci_lo":0.7473462301587301,"score_ci_hi":0.8825892857142857,"top100":82,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8237847222222221,"hallu_rate_mean":0.85,"hallu_prevalence":1,"hallu_top100":78,"inter_rater_agreement":0.7507936507936508,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":25737,"model_cost_usd":0.41731},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.6655886243386244,"score_ci_lo":0.5842460317460317,"score_ci_hi":0.7486970899470899,"top100":67,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6655886243386244,"hallu_rate_mean":1.8833333333333333,"hallu_prevalence":1,"hallu_top100":52,"inter_rater_agreement":0.7004761904761904,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":39545,"model_cost_usd":0.61864},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.8268419312169312,"score_ci_lo":0.7726951058201058,"score_ci_hi":0.876577380952381,"top100":83,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8268419312169312,"hallu_rate_mean":1.1333333333333333,"hallu_prevalence":1,"hallu_top100":71,"inter_rater_agreement":0.7792857142857144,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":69764,"model_cost_usd":1.102105},{"model_id":"grok-4.3","domain":"jura","score_mean":0.480062358276644,"score_ci_lo":0.42194727891156464,"score_ci_hi":0.5383531746031746,"top100":48,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.480062358276644,"hallu_rate_mean":0.7785714285714286,"hallu_prevalence":0.6285714285714286,"hallu_top100":80,"inter_rater_agreement":0.6595351473922902,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4471,"model_cost_usd":0.033129},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5337797619047618,"score_ci_lo":0.46125992063492066,"score_ci_hi":0.6063988095238095,"top100":53,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5337797619047618,"hallu_rate_mean":0.6625,"hallu_prevalence":0.75,"hallu_top100":83,"inter_rater_agreement":0.7006944444444445,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3795,"model_cost_usd":0.020806},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.25875661375661374,"score_ci_lo":0.19453703703703704,"score_ci_hi":0.32607142857142857,"top100":26,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.25875661375661374,"hallu_rate_mean":2.0833333333333335,"hallu_prevalence":1,"hallu_top100":47,"inter_rater_agreement":0.5367195767195767,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3312,"model_cost_usd":0.017699},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.5101190476190476,"score_ci_lo":0.44893849206349207,"score_ci_hi":0.5702083333333333,"top100":51,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5101190476190476,"hallu_rate_mean":1.2,"hallu_prevalence":0.8333333333333334,"hallu_top100":70,"inter_rater_agreement":0.5973015873015872,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":7998,"model_cost_usd":0.040362},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4690702947845805,"score_ci_lo":0.39168650793650794,"score_ci_hi":0.5428401360544218,"top100":47,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4690702947845805,"hallu_rate_mean":2.0142857142857142,"hallu_prevalence":0.8285714285714286,"hallu_top100":49,"inter_rater_agreement":0.6361224489795918,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10513,"model_cost_usd":0.089742},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6584077380952381,"score_ci_lo":0.5936755952380952,"score_ci_hi":0.7243055555555555,"top100":66,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6584077380952381,"hallu_rate_mean":1.1125,"hallu_prevalence":0.9,"hallu_top100":72,"inter_rater_agreement":0.7218253968253968,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10089,"model_cost_usd":0.073872},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.24625,"score_ci_lo":0.1726388888888889,"score_ci_hi":0.32208333333333333,"top100":25,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.24625,"hallu_rate_mean":2.9555555555555557,"hallu_prevalence":1,"hallu_top100":26,"inter_rater_agreement":0.49301587301587296,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":6996,"model_cost_usd":0.05312},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5729761904761904,"score_ci_lo":0.4984126984126984,"score_ci_hi":0.6432142857142857,"top100":57,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5729761904761904,"hallu_rate_mean":2.1083333333333334,"hallu_prevalence":0.9666666666666667,"hallu_top100":47,"inter_rater_agreement":0.5383862433862434,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18770,"model_cost_usd":0.13636}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.8075863095238095,"score_ci_lo":0.780157738095238,"score_ci_hi":0.8328521825396825,"top100":81,"n_cells":100,"hallu_rate_mean_across_domains":1.635,"hallu_top100_across_domains":59},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.739451388888889,"score_ci_lo":0.7083829365079365,"score_ci_hi":0.7660892857142856,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.1,"hallu_top100_across_domains":72},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5363422619047619,"score_ci_lo":0.4974305555555556,"score_ci_hi":0.5715376984126984,"top100":54,"n_cells":100,"hallu_rate_mean_across_domains":0.8075,"hallu_top100_across_domains":79},{"model_id":"gpt-5","score_mean_across_domains":0.7728164682539682,"score_ci_lo":0.736905753968254,"score_ci_hi":0.8057460317460317,"top100":77,"n_cells":100,"hallu_rate_mean_across_domains":1.105,"hallu_top100_across_domains":72},{"model_id":"grok-4.3","score_mean_across_domains":0.4666269841269841,"score_ci_lo":0.42845932539682535,"score_ci_hi":0.5047609126984127,"top100":47,"n_cells":100,"hallu_rate_mean_across_domains":1.0775,"hallu_top100_across_domains":73},{"model_id":"mistral-large-2","score_mean_across_domains":0.5046865079365079,"score_ci_lo":0.4552708333333333,"score_ci_hi":0.5496825396825397,"top100":50,"n_cells":100,"hallu_rate_mean_across_domains":2.0033333333333334,"hallu_top100_across_domains":49}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8775,"oracle_top100":88,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8402,"best_single_model_top100":84},{"domain":"medizin","n_items":20,"oracle_mean":0.9078,"oracle_top100":91,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8563,"best_single_model_top100":86},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.8124,"oracle_top100":81,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7666,"best_single_model_top100":77},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8744,"oracle_top100":87,"best_single_model_id":"gpt-5","best_single_model_mean":0.8268,"best_single_model_top100":83}],"cost":{"model_calls_usd":10.203,"judge_calls_usd":62.0398,"total_usd":72.2428,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1660984,"tokens_out_total":363468,"cost_usd":17.39162},{"judge_id":"claude-opus-4-8","tokens_in_total":1657984,"tokens_out_total":453424,"cost_usd":19.62552},{"judge_id":"gpt-5","tokens_in_total":858944,"tokens_out_total":1101200,"cost_usd":20.81272},{"judge_id":"mistral-large-2","tokens_in_total":918465,"tokens_out_total":395508,"cost_usd":4.209978}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":866.91,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260607T181746-23cbb6/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260607T181746-23cbb6/progress.jsonl","cost_total_usd":72.2428}