{"run_id":"run-20260516T101429-0435c4","started_at":1778926469,"summary":{"run_id":"run-20260516T101429-0435c4","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778926469.2122025,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8516038359788359,"score_ci_lo":0.7963789682539683,"score_ci_hi":0.9039351851851851,"top100":85,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.8516038359788359,"hallu_rate_mean":0.6666666666666666,"hallu_prevalence":0.625,"hallu_top100":83,"inter_rater_agreement":0.9248511904761905,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6426,"model_cost_usd":0.57648},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8132716049382717,"score_ci_lo":0.6970164609053497,"score_ci_hi":0.8930041152263374,"top100":81,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8132716049382717,"hallu_rate_mean":1.1111111111111112,"hallu_prevalence":0.8888888888888888,"hallu_top100":72,"inter_rater_agreement":0.8425925925925926,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9213,"model_cost_usd":0.77355},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6897321428571428,"score_ci_lo":0.6145833333333334,"score_ci_hi":0.7648809523809523,"top100":69,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6897321428571428,"hallu_rate_mean":2.7916666666666665,"hallu_prevalence":1,"hallu_top100":30,"inter_rater_agreement":0.7799107142857142,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":9963,"model_cost_usd":0.83391},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7708333333333334,"score_ci_lo":0.6833333333333333,"score_ci_hi":0.8416666666666667,"top100":77,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7708333333333334,"hallu_rate_mean":2.916666666666667,"hallu_prevalence":1,"hallu_top100":27,"inter_rater_agreement":0.7875,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":15156,"model_cost_usd":1.26126},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5327380952380952,"score_ci_lo":0.4328703703703704,"score_ci_hi":0.648478835978836,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5327380952380952,"hallu_rate_mean":0.41666666666666663,"hallu_prevalence":0.25,"hallu_top100":89,"inter_rater_agreement":0.6865079365079365,"n_judges":3,"tokens_in_total":3570,"tokens_out_total":1695,"model_cost_usd":0.021413},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.5174897119341564,"score_ci_lo":0.44547325102880664,"score_ci_hi":0.5797325102880658,"top100":52,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5174897119341564,"hallu_rate_mean":0.5555555555555556,"hallu_prevalence":0.7777777777777778,"hallu_top100":86,"inter_rater_agreement":0.6867283950617283,"n_judges":3,"tokens_in_total":3142,"tokens_out_total":2079,"model_cost_usd":0.024718},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.2604166666666667,"score_ci_lo":0.15625,"score_ci_hi":0.4010416666666667,"top100":26,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.2604166666666667,"hallu_rate_mean":1.9583333333333335,"hallu_prevalence":0.875,"hallu_top100":51,"inter_rater_agreement":0.6526785714285714,"n_judges":3,"tokens_in_total":3170,"tokens_out_total":1708,"model_cost_usd":0.021043},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5041666666666667,"score_ci_lo":0.41250000000000003,"score_ci_hi":0.5916666666666667,"top100":50,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5041666666666667,"hallu_rate_mean":1.5416666666666667,"hallu_prevalence":0.875,"hallu_top100":61,"inter_rater_agreement":0.625,"n_judges":3,"tokens_in_total":4357,"tokens_out_total":3226,"model_cost_usd":0.037706},{"model_id":"gpt-5","domain":"jura","score_mean":0.6750992063492063,"score_ci_lo":0.5636574074074074,"score_ci_hi":0.7857142857142857,"top100":68,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6750992063492063,"hallu_rate_mean":1,"hallu_prevalence":0.75,"hallu_top100":75,"inter_rater_agreement":0.6557539682539683,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":13305,"model_cost_usd":0.215845},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8503086419753086,"score_ci_lo":0.793724279835391,"score_ci_hi":0.9022633744855967,"top100":85,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8503086419753086,"hallu_rate_mean":1.2592592592592593,"hallu_prevalence":1,"hallu_top100":68,"inter_rater_agreement":0.8009259259259259,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":14069,"model_cost_usd":0.22601},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.606845238095238,"score_ci_lo":0.47916666666666663,"score_ci_hi":0.7272321428571429,"top100":61,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.606845238095238,"hallu_rate_mean":2.375,"hallu_prevalence":1,"hallu_top100":40,"inter_rater_agreement":0.8040178571428571,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":23341,"model_cost_usd":0.365115},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7916666666666666,"score_ci_lo":0.7125,"score_ci_hi":0.8666666666666667,"top100":79,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7916666666666666,"hallu_rate_mean":1.8333333333333333,"hallu_prevalence":1,"hallu_top100":54,"inter_rater_agreement":0.7,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":21269,"model_cost_usd":0.340095},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4767691798941799,"score_ci_lo":0.34887566137566134,"score_ci_hi":0.5906911375661376,"top100":48,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4767691798941799,"hallu_rate_mean":0.08333333333333333,"hallu_prevalence":0.125,"hallu_top100":97,"inter_rater_agreement":0.6847718253968254,"n_judges":3,"tokens_in_total":4643,"tokens_out_total":1031,"model_cost_usd":0.008381},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5246913580246914,"score_ci_lo":0.46913580246913583,"score_ci_hi":0.5895061728395061,"top100":52,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5246913580246914,"hallu_rate_mean":0.5555555555555556,"hallu_prevalence":0.4444444444444444,"hallu_top100":86,"inter_rater_agreement":0.7391975308641975,"n_judges":3,"tokens_in_total":4263,"tokens_out_total":1932,"model_cost_usd":0.010159},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.17619047619047618,"score_ci_lo":0.08020833333333333,"score_ci_hi":0.28169642857142857,"top100":18,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.17619047619047618,"hallu_rate_mean":2.125,"hallu_prevalence":0.875,"hallu_top100":46,"inter_rater_agreement":0.5933035714285715,"n_judges":3,"tokens_in_total":4343,"tokens_out_total":1990,"model_cost_usd":0.010404},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.44166666666666665,"score_ci_lo":0.2791666666666667,"score_ci_hi":0.6083333333333333,"top100":44,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.44166666666666665,"hallu_rate_mean":2.125,"hallu_prevalence":1,"hallu_top100":46,"inter_rater_agreement":0.6125,"n_judges":3,"tokens_in_total":5722,"tokens_out_total":3254,"model_cost_usd":0.015287},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.49446097883597884,"score_ci_lo":0.40707671957671954,"score_ci_hi":0.5844080687830688,"top100":49,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.49446097883597884,"hallu_rate_mean":1,"hallu_prevalence":0.875,"hallu_top100":75,"inter_rater_agreement":0.6475694444444444,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2442,"model_cost_usd":0.022048},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6831275720164609,"score_ci_lo":0.5895061728395061,"score_ci_hi":0.7746913580246914,"top100":68,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6831275720164609,"hallu_rate_mean":1.2222222222222223,"hallu_prevalence":1,"hallu_top100":69,"inter_rater_agreement":0.7345679012345678,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5038,"model_cost_usd":0.036614},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.13095238095238096,"score_ci_lo":0.07410714285714286,"score_ci_hi":0.18958333333333333,"top100":13,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.13095238095238096,"hallu_rate_mean":3.625,"hallu_prevalence":1,"hallu_top100":9,"inter_rater_agreement":0.5169642857142858,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3964,"model_cost_usd":0.030302},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.4291666666666667,"score_ci_lo":0.3041666666666667,"score_ci_hi":0.5541666666666667,"top100":43,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4291666666666667,"hallu_rate_mean":3.1666666666666665,"hallu_prevalence":1,"hallu_top100":20,"inter_rater_agreement":0.475,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6616,"model_cost_usd":0.048668}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7823272406605739,"score_ci_lo":0.7307760141093473,"score_ci_hi":0.8275613275613276,"top100":78,"n_cells":33,"hallu_rate_mean_across_domains":1.8484848484848484,"hallu_top100_across_domains":53},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.45563572230238897,"score_ci_lo":0.39536235369568706,"score_ci_hi":0.5195967612634279,"top100":46,"n_cells":33,"hallu_rate_mean_across_domains":1.101010101010101,"hallu_top100_across_domains":72},{"model_id":"gpt-5","score_mean_across_domains":0.7345959595959596,"score_ci_lo":0.6757856341189674,"score_ci_hi":0.7908409491742825,"top100":73,"n_cells":33,"hallu_rate_mean_across_domains":1.606060606060606,"hallu_top100_across_domains":59},{"model_id":"grok-4.3","score_mean_across_domains":0.4084616001282668,"score_ci_lo":0.32836700336700336,"score_ci_hi":0.48505291005290996,"top100":41,"n_cells":33,"hallu_rate_mean_across_domains":1.202020202020202,"hallu_top100_across_domains":69},{"model_id":"mistral-large-2","score_mean_across_domains":0.44196328362995024,"score_ci_lo":0.36402918069584733,"score_ci_hi":0.5217853134519802,"top100":44,"n_cells":33,"hallu_rate_mean_across_domains":2.222222222222222,"hallu_top100_across_domains":44}],"cost":{"model_calls_usd":4.879,"judge_calls_usd":23.7661,"total_usd":28.6451,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":519594,"tokens_out_total":110929,"cost_usd":16.113585},{"judge_id":"gpt-5","tokens_in_total":270170,"tokens_out_total":334704,"cost_usd":6.37141},{"judge_id":"mistral-large-2","tokens_in_total":288564,"tokens_out_total":117327,"cost_usd":1.28109}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":1041.64,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260516T101429-0435c4/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260516T101429-0435c4/progress.jsonl","cost_total_usd":28.6451}