{"run_id":"run-20260618T165158-d2543a","started_at":1781801518,"summary":{"run_id":"run-20260618T165158-d2543a","question_bank_version":"2026-05-17-v1","code_version":"0.2.6","started_at":1781801518.584576,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8409410430839002,"score_ci_lo":0.7976445578231293,"score_ci_hi":0.8800566893424037,"top100":84,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8409410430839002,"hallu_rate_mean":0.9142857142857143,"hallu_prevalence":0.7714285714285715,"hallu_top100":77,"inter_rater_agreement":0.8443764172335602,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":28983,"model_cost_usd":0.839325},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8478174603174603,"score_ci_lo":0.790798611111111,"score_ci_hi":0.9033730158730158,"top100":85,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8478174603174603,"hallu_rate_mean":1.1,"hallu_prevalence":0.95,"hallu_top100":72,"inter_rater_agreement":0.881547619047619,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19688,"model_cost_usd":0.549675},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6995171957671958,"score_ci_lo":0.6365079365079365,"score_ci_hi":0.7668584656084657,"top100":70,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6995171957671958,"hallu_rate_mean":2.4833333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":37,"inter_rater_agreement":0.7122486772486772,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":18102,"model_cost_usd":0.5016},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7766534391534391,"score_ci_lo":0.7264649470899471,"score_ci_hi":0.8289814814814814,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7766534391534391,"hallu_rate_mean":2.2666666666666666,"hallu_prevalence":1,"hallu_top100":43,"inter_rater_agreement":0.7037433862433863,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":40275,"model_cost_usd":1.118185},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7792006802721089,"score_ci_lo":0.7163832199546486,"score_ci_hi":0.8329960317460318,"top100":78,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7792006802721089,"hallu_rate_mean":0.8071428571428572,"hallu_prevalence":0.7142857142857143,"hallu_top100":79,"inter_rater_agreement":0.7656916099773242,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":32103,"model_cost_usd":0.91645},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.7908730158730158,"score_ci_lo":0.7108630952380952,"score_ci_hi":0.859002976190476,"top100":79,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7908730158730158,"hallu_rate_mean":0.575,"hallu_prevalence":0.75,"hallu_top100":85,"inter_rater_agreement":0.7581349206349206,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":20480,"model_cost_usd":0.568975},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6858465608465608,"score_ci_lo":0.6259722222222222,"score_ci_hi":0.7434391534391535,"top100":69,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6858465608465608,"hallu_rate_mean":2.0166666666666666,"hallu_prevalence":1,"hallu_top100":49,"inter_rater_agreement":0.7576984126984126,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":21317,"model_cost_usd":0.5816},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.690433201058201,"score_ci_lo":0.6313822751322751,"score_ci_hi":0.7468816137566138,"top100":69,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.690433201058201,"hallu_rate_mean":1.75,"hallu_prevalence":1,"hallu_top100":56,"inter_rater_agreement":0.7023941798941798,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":46368,"model_cost_usd":1.26976},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6118367346938776,"score_ci_lo":0.550765306122449,"score_ci_hi":0.6670946712018141,"top100":61,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6118367346938776,"hallu_rate_mean":0.2714285714285714,"hallu_prevalence":0.2857142857142857,"hallu_top100":93,"inter_rater_agreement":0.7360090702947846,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":6796,"model_cost_usd":0.084082},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5942708333333333,"score_ci_lo":0.517435515873016,"score_ci_hi":0.6632936507936508,"top100":59,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5942708333333333,"hallu_rate_mean":0.4875,"hallu_prevalence":0.55,"hallu_top100":87,"inter_rater_agreement":0.7350198412698413,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4217,"model_cost_usd":0.050333},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.3452314814814815,"score_ci_lo":0.24541666666666667,"score_ci_hi":0.45652777777777775,"top100":35,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3452314814814815,"hallu_rate_mean":1.95,"hallu_prevalence":0.8666666666666667,"hallu_top100":51,"inter_rater_agreement":0.5975132275132274,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3186,"model_cost_usd":0.038635},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.4802215608465608,"score_ci_lo":0.41235449735449736,"score_ci_hi":0.5522916666666667,"top100":48,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4802215608465608,"hallu_rate_mean":1.1583333333333334,"hallu_prevalence":0.8666666666666667,"hallu_top100":71,"inter_rater_agreement":0.5509126984126984,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":8540,"model_cost_usd":0.099831},{"model_id":"gpt-5","domain":"jura","score_mean":0.7365334467120181,"score_ci_lo":0.6579365079365079,"score_ci_hi":0.7989484126984127,"top100":74,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7365334467120181,"hallu_rate_mean":0.9571428571428572,"hallu_prevalence":0.8857142857142857,"hallu_top100":76,"inter_rater_agreement":0.7589002267573696,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":59990,"model_cost_usd":0.9591},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8361855158730158,"score_ci_lo":0.7747767857142857,"score_ci_hi":0.8933035714285713,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8361855158730158,"hallu_rate_mean":1.025,"hallu_prevalence":0.9,"hallu_top100":74,"inter_rater_agreement":0.7836309523809524,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":26670,"model_cost_usd":0.431305},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.6158201058201058,"score_ci_lo":0.49018518518518517,"score_ci_hi":0.7333333333333333,"top100":62,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6158201058201058,"hallu_rate_mean":2.2,"hallu_prevalence":1,"hallu_top100":44,"inter_rater_agreement":0.6481746031746032,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":39051,"model_cost_usd":0.61123},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.8016302910052909,"score_ci_lo":0.748535052910053,"score_ci_hi":0.8461607142857143,"top100":80,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8016302910052909,"hallu_rate_mean":1.15,"hallu_prevalence":0.9666666666666667,"hallu_top100":71,"inter_rater_agreement":0.6409920634920635,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":68184,"model_cost_usd":1.078405},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4162953514739229,"score_ci_lo":0.3446286848072562,"score_ci_hi":0.4876190476190476,"top100":42,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4162953514739229,"hallu_rate_mean":1.2428571428571429,"hallu_prevalence":0.6571428571428571,"hallu_top100":68,"inter_rater_agreement":0.6785827664399093,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4066,"model_cost_usd":0.032116},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5302083333333333,"score_ci_lo":0.45513392857142854,"score_ci_hi":0.6025545634920635,"top100":53,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5302083333333333,"hallu_rate_mean":0.35,"hallu_prevalence":0.4,"hallu_top100":91,"inter_rater_agreement":0.6801587301587302,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3913,"model_cost_usd":0.021101},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.23328703703703704,"score_ci_lo":0.1437037037037037,"score_ci_hi":0.3222222222222222,"top100":23,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.23328703703703704,"hallu_rate_mean":2.35,"hallu_prevalence":1,"hallu_top100":41,"inter_rater_agreement":0.5087566137566137,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3401,"model_cost_usd":0.017921},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.5149074074074075,"score_ci_lo":0.4319642857142857,"score_ci_hi":0.5896130952380952,"top100":51,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5149074074074075,"hallu_rate_mean":1.35,"hallu_prevalence":0.7666666666666667,"hallu_top100":66,"inter_rater_agreement":0.5647354497354498,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":8665,"model_cost_usd":0.04203},{"model_id":"kimi-k2.6","domain":"jura","score_mean":0.5721910430839002,"score_ci_lo":0.48689342403628116,"score_ci_hi":0.6509722222222222,"top100":57,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5721910430839002,"hallu_rate_mean":1.8142857142857143,"hallu_prevalence":0.8285714285714286,"hallu_top100":54,"inter_rater_agreement":0.68202947845805,"n_judges":4,"tokens_in_total":15188,"tokens_out_total":248814,"model_cost_usd":1.009685},{"model_id":"kimi-k2.6","domain":"medizin","score_mean":0.6625744047619048,"score_ci_lo":0.5894097222222222,"score_ci_hi":0.7405753968253969,"top100":66,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6625744047619048,"hallu_rate_mean":0.95,"hallu_prevalence":0.95,"hallu_top100":76,"inter_rater_agreement":0.6195436507936508,"n_judges":4,"tokens_in_total":7536,"tokens_out_total":106790,"model_cost_usd":0.434319},{"model_id":"kimi-k2.6","domain":"steuerrecht","score_mean":0.36513227513227514,"score_ci_lo":0.2622685185185185,"score_ci_hi":0.47882275132275137,"top100":37,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.36513227513227514,"hallu_rate_mean":3.316666666666667,"hallu_prevalence":1,"hallu_top100":17,"inter_rater_agreement":0.5987301587301588,"n_judges":4,"tokens_in_total":6443,"tokens_out_total":135443,"model_cost_usd":0.547893},{"model_id":"kimi-k2.6","domain":"wirtschaftsrecht","score_mean":0.5809391534391535,"score_ci_lo":0.4960482804232804,"score_ci_hi":0.6646031746031745,"top100":58,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5809391534391535,"hallu_rate_mean":2.875,"hallu_prevalence":1,"hallu_top100":28,"inter_rater_agreement":0.532037037037037,"n_judges":4,"tokens_in_total":14354,"tokens_out_total":234232,"model_cost_usd":0.950564},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.44136054421768706,"score_ci_lo":0.3625510204081633,"score_ci_hi":0.5195634920634921,"top100":44,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.44136054421768706,"hallu_rate_mean":2.142857142857143,"hallu_prevalence":0.8285714285714286,"hallu_top100":46,"inter_rater_agreement":0.6035034013605441,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10608,"model_cost_usd":0.090312},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6789186507936508,"score_ci_lo":0.6140625,"score_ci_hi":0.7430803571428571,"top100":68,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6789186507936508,"hallu_rate_mean":1.15,"hallu_prevalence":0.95,"hallu_top100":71,"inter_rater_agreement":0.7514880952380952,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10088,"model_cost_usd":0.073866},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.2769642857142857,"score_ci_lo":0.17716931216931217,"score_ci_hi":0.38277777777777783,"top100":28,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2769642857142857,"hallu_rate_mean":3,"hallu_prevalence":1,"hallu_top100":25,"inter_rater_agreement":0.6042063492063492,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":6987,"model_cost_usd":0.053066},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5841335978835979,"score_ci_lo":0.5126587301587302,"score_ci_hi":0.6444113756613756,"top100":58,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5841335978835979,"hallu_rate_mean":2.3,"hallu_prevalence":0.9333333333333333,"hallu_top100":42,"inter_rater_agreement":0.5467724867724867,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18366,"model_cost_usd":0.133936}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.8018164682539682,"score_ci_lo":0.7744920634920635,"score_ci_hi":0.8285833333333333,"top100":80,"n_cells":100,"hallu_rate_mean_across_domains":1.5925,"hallu_top100_across_domains":60},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.7409017857142857,"score_ci_lo":0.7040793650793651,"score_ci_hi":0.7751726190476191,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.225,"hallu_top100_across_domains":69},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5288482142857143,"score_ci_lo":0.48621031746031745,"score_ci_hi":0.5692827380952381,"top100":53,"n_cells":100,"hallu_rate_mean_across_domains":0.8325,"hallu_top100_across_domains":79},{"model_id":"gpt-5","score_mean_across_domains":0.7578859126984128,"score_ci_lo":0.7178263888888888,"score_ci_hi":0.7944960317460317,"top100":76,"n_cells":100,"hallu_rate_mean_across_domains":1.215,"hallu_top100_across_domains":69},{"model_id":"grok-4.3","score_mean_across_domains":0.44121031746031747,"score_ci_lo":0.3925486111111111,"score_ci_hi":0.48564583333333333,"top100":44,"n_cells":100,"hallu_rate_mean_across_domains":1.2625,"hallu_top100_across_domains":68},{"model_id":"kimi-k2.6","score_mean_across_domains":0.5618333333333333,"score_ci_lo":0.5137857142857143,"score_ci_hi":0.6070069444444445,"top100":56,"n_cells":100,"hallu_rate_mean_across_domains":2.185,"hallu_top100_across_domains":45},{"model_id":"mistral-large-2","score_mean_across_domains":0.5070446428571429,"score_ci_lo":0.45465079365079364,"score_ci_hi":0.5531924603174603,"top100":51,"n_cells":100,"hallu_rate_mean_across_domains":2.12,"hallu_top100_across_domains":47}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.873,"oracle_top100":87,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8409,"best_single_model_top100":84},{"domain":"medizin","n_items":20,"oracle_mean":0.9156,"oracle_top100":92,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8478,"best_single_model_top100":85},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7939,"oracle_top100":79,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.6995,"best_single_model_top100":70},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8582,"oracle_top100":86,"best_single_model_id":"gpt-5","best_single_model_mean":0.8016,"best_single_model_top100":80}],"cost":{"model_calls_usd":13.1053,"judge_calls_usd":73.9649,"total_usd":87.0702,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1937288,"tokens_out_total":434729,"cost_usd":20.554665},{"judge_id":"claude-opus-4-8","tokens_in_total":1933788,"tokens_out_total":555893,"cost_usd":23.566265},{"judge_id":"gpt-5","tokens_in_total":1000806,"tokens_out_total":1324622,"cost_usd":24.87336},{"judge_id":"mistral-large-2","tokens_in_total":1073753,"tokens_out_total":470509,"cost_usd":4.97056}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":1044.84,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260618T165158-d2543a/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260618T165158-d2543a/progress.jsonl","cost_total_usd":87.0702}