{"run_id":"run-20260615T043639-30c761","started_at":1781498199,"summary":{"run_id":"run-20260615T043639-30c761","question_bank_version":"2026-05-17-v1","code_version":"0.2.4","started_at":1781498199.7747452,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8031349206349206,"score_ci_lo":0.7470238095238095,"score_ci_hi":0.8509722222222221,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8031349206349206,"hallu_rate_mean":0.9928571428571429,"hallu_prevalence":0.7428571428571429,"hallu_top100":75,"inter_rater_agreement":0.8138548752834467,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":27656,"model_cost_usd":0.80615},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8499751984126984,"score_ci_lo":0.7959821428571429,"score_ci_hi":0.9048363095238094,"top100":85,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8499751984126984,"hallu_rate_mean":1.1875,"hallu_prevalence":1,"hallu_top100":70,"inter_rater_agreement":0.7530753968253968,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19608,"model_cost_usd":0.547675},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7484986772486771,"score_ci_lo":0.6781944444444444,"score_ci_hi":0.8125462962962964,"top100":75,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7484986772486771,"hallu_rate_mean":2.3,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.6846296296296296,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":19621,"model_cost_usd":0.539575},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7679927248677249,"score_ci_lo":0.7159523809523809,"score_ci_hi":0.8160482804232804,"top100":77,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7679927248677249,"hallu_rate_mean":2.658333333333333,"hallu_prevalence":1,"hallu_top100":33,"inter_rater_agreement":0.7248941798941799,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":41639,"model_cost_usd":1.152285},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7991043083900227,"score_ci_lo":0.7522137188208616,"score_ci_hi":0.8426332199546486,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7991043083900227,"hallu_rate_mean":0.5714285714285714,"hallu_prevalence":0.6571428571428571,"hallu_top100":85,"inter_rater_agreement":0.7743424036281179,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":32193,"model_cost_usd":0.9187},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.788764880952381,"score_ci_lo":0.7115079365079364,"score_ci_hi":0.8679315476190477,"top100":79,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.788764880952381,"hallu_rate_mean":0.65,"hallu_prevalence":0.75,"hallu_top100":83,"inter_rater_agreement":0.6773809523809524,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":19939,"model_cost_usd":0.55545},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6621230158730159,"score_ci_lo":0.6018452380952382,"score_ci_hi":0.7261441798941799,"top100":66,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6621230158730159,"hallu_rate_mean":1.5833333333333333,"hallu_prevalence":1,"hallu_top100":60,"inter_rater_agreement":0.7272486772486773,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":19772,"model_cost_usd":0.542975},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.6844576719576719,"score_ci_lo":0.6243551587301588,"score_ci_hi":0.7415773809523809,"top100":68,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6844576719576719,"hallu_rate_mean":1.6,"hallu_prevalence":0.9333333333333333,"hallu_top100":60,"inter_rater_agreement":0.5948015873015873,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":45376,"model_cost_usd":1.24496},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.5895068027210885,"score_ci_lo":0.5372902494331065,"score_ci_hi":0.6413492063492063,"top100":59,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5895068027210885,"hallu_rate_mean":0.37142857142857144,"hallu_prevalence":0.4,"hallu_top100":90,"inter_rater_agreement":0.7090589569160998,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":6739,"model_cost_usd":0.083513},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.6145089285714286,"score_ci_lo":0.5449156746031746,"score_ci_hi":0.6809027777777777,"top100":61,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6145089285714286,"hallu_rate_mean":0.2,"hallu_prevalence":0.4,"hallu_top100":95,"inter_rater_agreement":0.7039682539682539,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4089,"model_cost_usd":0.049052},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.37680555555555556,"score_ci_lo":0.27425925925925926,"score_ci_hi":0.47976851851851854,"top100":38,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.37680555555555556,"hallu_rate_mean":1.5333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":61,"inter_rater_agreement":0.5341798941798942,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3343,"model_cost_usd":0.040205},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.5219742063492063,"score_ci_lo":0.4664384920634921,"score_ci_hi":0.5829662698412699,"top100":52,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5219742063492063,"hallu_rate_mean":1.0416666666666667,"hallu_prevalence":0.8,"hallu_top100":73,"inter_rater_agreement":0.6076455026455025,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":8331,"model_cost_usd":0.097741},{"model_id":"gpt-5","domain":"jura","score_mean":0.7290986394557822,"score_ci_lo":0.6476899092970522,"score_ci_hi":0.7944586167800454,"top100":73,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7290986394557822,"hallu_rate_mean":1.1142857142857143,"hallu_prevalence":0.8,"hallu_top100":72,"inter_rater_agreement":0.7406462585034014,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":56727,"model_cost_usd":0.910155},{"model_id":"gpt-5","domain":"medizin","score_mean":0.7995535714285714,"score_ci_lo":0.7322172619047619,"score_ci_hi":0.8605902777777779,"top100":80,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7995535714285714,"hallu_rate_mean":0.8666666666666666,"hallu_prevalence":0.8,"hallu_top100":78,"inter_rater_agreement":0.770734126984127,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":26136,"model_cost_usd":0.423295},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5888690476190476,"score_ci_lo":0.46324074074074073,"score_ci_hi":0.7027380952380953,"top100":59,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5888690476190476,"hallu_rate_mean":1.9666666666666666,"hallu_prevalence":1,"hallu_top100":50,"inter_rater_agreement":0.5515873015873016,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":39441,"model_cost_usd":0.61708},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7992162698412699,"score_ci_lo":0.7465575396825397,"score_ci_hi":0.8473710317460318,"top100":80,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7992162698412699,"hallu_rate_mean":1.2166666666666666,"hallu_prevalence":0.9,"hallu_top100":69,"inter_rater_agreement":0.6818650793650795,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":67355,"model_cost_usd":1.06597},{"model_id":"grok-4.3","domain":"jura","score_mean":0.5061763038548752,"score_ci_lo":0.45698979591836736,"score_ci_hi":0.5508390022675737,"top100":51,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5061763038548752,"hallu_rate_mean":0.35714285714285715,"hallu_prevalence":0.4,"hallu_top100":91,"inter_rater_agreement":0.6402494331065759,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4415,"model_cost_usd":0.032989},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5236855158730159,"score_ci_lo":0.469890873015873,"score_ci_hi":0.5790922619047618,"top100":52,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5236855158730159,"hallu_rate_mean":0.3875,"hallu_prevalence":0.55,"hallu_top100":90,"inter_rater_agreement":0.7482142857142857,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3775,"model_cost_usd":0.020756},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.26292989417989415,"score_ci_lo":0.17217592592592593,"score_ci_hi":0.362989417989418,"top100":26,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.26292989417989415,"hallu_rate_mean":2.65,"hallu_prevalence":1,"hallu_top100":33,"inter_rater_agreement":0.5798677248677249,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3496,"model_cost_usd":0.018159},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.5299900793650794,"score_ci_lo":0.4791269841269841,"score_ci_hi":0.583234126984127,"top100":53,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5299900793650794,"hallu_rate_mean":1.075,"hallu_prevalence":0.9,"hallu_top100":73,"inter_rater_agreement":0.6119312169312169,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":8465,"model_cost_usd":0.04153},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.41554705215419496,"score_ci_lo":0.33888888888888885,"score_ci_hi":0.49452664399092966,"top100":42,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.41554705215419496,"hallu_rate_mean":1.9071428571428573,"hallu_prevalence":0.8,"hallu_top100":52,"inter_rater_agreement":0.6588548752834467,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10565,"model_cost_usd":0.090054},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6921130952380953,"score_ci_lo":0.6349206349206349,"score_ci_hi":0.7463789682539683,"top100":69,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6921130952380953,"hallu_rate_mean":1.1375,"hallu_prevalence":0.95,"hallu_top100":71,"inter_rater_agreement":0.7431547619047618,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10422,"model_cost_usd":0.07587},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.3043518518518519,"score_ci_lo":0.20259259259259257,"score_ci_hi":0.40921296296296295,"top100":30,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3043518518518519,"hallu_rate_mean":3.466666666666667,"hallu_prevalence":1,"hallu_top100":13,"inter_rater_agreement":0.5650529100529101,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":7133,"model_cost_usd":0.053942},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5384126984126983,"score_ci_lo":0.4766765873015873,"score_ci_hi":0.6015178571428572,"top100":54,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5384126984126983,"hallu_rate_mean":2.2916666666666665,"hallu_prevalence":0.9666666666666667,"hallu_top100":42,"inter_rater_agreement":0.5229761904761905,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18969,"model_cost_usd":0.137554}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7937648809523808,"score_ci_lo":0.7643561507936508,"score_ci_hi":0.8201259920634921,"top100":79,"n_cells":100,"hallu_rate_mean_across_domains":1.7275,"hallu_top100_across_domains":56},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.7420952380952381,"score_ci_lo":0.7109930555555556,"score_ci_hi":0.7727450396825397,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.0475,"hallu_top100_across_domains":73},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.542342261904762,"score_ci_lo":0.5045595238095238,"score_ci_hi":0.5787599206349207,"top100":54,"n_cells":100,"hallu_rate_mean_across_domains":0.7125,"hallu_top100_across_domains":82},{"model_id":"gpt-5","score_mean_across_domains":0.7431904761904762,"score_ci_lo":0.7038839285714286,"score_ci_hi":0.7816488095238096,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.2233333333333334,"hallu_top100_across_domains":69},{"model_id":"grok-4.3","score_mean_across_domains":0.4803353174603175,"score_ci_lo":0.4440496031746031,"score_ci_hi":0.5166031746031746,"top100":48,"n_cells":100,"hallu_rate_mean_across_domains":0.9225,"hallu_top100_across_domains":76},{"model_id":"mistral-large-2","score_mean_across_domains":0.4910406746031746,"score_ci_lo":0.44292956349206347,"score_ci_hi":0.5384345238095238,"top100":49,"n_cells":100,"hallu_rate_mean_across_domains":2.1025,"hallu_top100_across_domains":47}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8585,"oracle_top100":86,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8031,"best_single_model_top100":80},{"domain":"medizin","n_items":20,"oracle_mean":0.9047,"oracle_top100":90,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.85,"best_single_model_top100":85},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.8089,"oracle_top100":81,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7485,"best_single_model_top100":75},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8626,"oracle_top100":86,"best_single_model_id":"gpt-5","best_single_model_mean":0.7992,"best_single_model_top100":80}],"cost":{"model_calls_usd":10.0656,"judge_calls_usd":61.9485,"total_usd":72.0142,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1655554,"tokens_out_total":363395,"cost_usd":17.362645},{"judge_id":"claude-opus-4-8","tokens_in_total":1652554,"tokens_out_total":453595,"cost_usd":19.602645},{"judge_id":"gpt-5","tokens_in_total":856728,"tokens_out_total":1101114,"cost_usd":20.80035},{"judge_id":"mistral-large-2","tokens_in_total":911196,"tokens_out_total":393417,"cost_usd":4.182894}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":864.17,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260615T043639-30c761/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260615T043639-30c761/progress.jsonl","cost_total_usd":72.0142}