{"run_id":"run-20260619T195640-4ce8c8","started_at":1781899000,"summary":{"run_id":"run-20260619T195640-4ce8c8","question_bank_version":"2026-05-17-v1","code_version":"0.2.6","started_at":1781899000.947256,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.787063492063492,"score_ci_lo":0.7339880952380953,"score_ci_hi":0.8376587301587302,"top100":79,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.787063492063492,"hallu_rate_mean":1.0357142857142858,"hallu_prevalence":0.8,"hallu_top100":74,"inter_rater_agreement":0.803140589569161,"n_judges":4,"tokens_in_total":22950,"tokens_out_total":27789,"model_cost_usd":0.809475},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8323908730158729,"score_ci_lo":0.7612599206349207,"score_ci_hi":0.8973214285714286,"top100":83,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8323908730158729,"hallu_rate_mean":1.325,"hallu_prevalence":0.95,"hallu_top100":66,"inter_rater_agreement":0.8220238095238095,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":19974,"model_cost_usd":0.556825},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7119775132275132,"score_ci_lo":0.6451388888888888,"score_ci_hi":0.7752050264550264,"top100":71,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7119775132275132,"hallu_rate_mean":2.1666666666666665,"hallu_prevalence":1,"hallu_top100":45,"inter_rater_agreement":0.7306613756613757,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":18027,"model_cost_usd":0.499725},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7582969576719577,"score_ci_lo":0.7134722222222223,"score_ci_hi":0.7976256613756614,"top100":76,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7582969576719577,"hallu_rate_mean":2.2583333333333333,"hallu_prevalence":0.9666666666666667,"hallu_top100":43,"inter_rater_agreement":0.6340740740740741,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":41418,"model_cost_usd":1.14676},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.7629024943310658,"score_ci_lo":0.7027976190476191,"score_ci_hi":0.8176643990929705,"top100":76,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7629024943310658,"hallu_rate_mean":0.75,"hallu_prevalence":0.6285714285714286,"hallu_top100":81,"inter_rater_agreement":0.722437641723356,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":31135,"model_cost_usd":0.89225},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.8193948412698413,"score_ci_lo":0.7673363095238095,"score_ci_hi":0.8725446428571428,"top100":82,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8193948412698413,"hallu_rate_mean":0.5875,"hallu_prevalence":0.85,"hallu_top100":85,"inter_rater_agreement":0.8134920634920635,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":19948,"model_cost_usd":0.555675},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.6707671957671958,"score_ci_lo":0.583042328042328,"score_ci_hi":0.7422486772486772,"top100":67,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6707671957671958,"hallu_rate_mean":1.55,"hallu_prevalence":1,"hallu_top100":61,"inter_rater_agreement":0.7966137566137567,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":21163,"model_cost_usd":0.57775},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.7096560846560848,"score_ci_lo":0.6518617724867725,"score_ci_hi":0.7667724867724868,"top100":71,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7096560846560848,"hallu_rate_mean":1.825,"hallu_prevalence":0.9666666666666667,"hallu_top100":54,"inter_rater_agreement":0.6464682539682539,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":45900,"model_cost_usd":1.25806},{"model_id":"gemini-3-1-pro","domain":"jura","score_mean":0.592202380952381,"score_ci_lo":0.5212301587301588,"score_ci_hi":0.6521230158730158,"top100":59,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.592202380952381,"hallu_rate_mean":0.75,"hallu_prevalence":0.6,"hallu_top100":81,"inter_rater_agreement":0.702437641723356,"n_judges":4,"tokens_in_total":12898,"tokens_out_total":6415,"model_cost_usd":0.102776},{"model_id":"gemini-3-1-pro","domain":"medizin","score_mean":0.6026041666666667,"score_ci_lo":0.5366071428571428,"score_ci_hi":0.6735367063492064,"top100":60,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6026041666666667,"hallu_rate_mean":0.4,"hallu_prevalence":0.6,"hallu_top100":90,"inter_rater_agreement":0.7100198412698413,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4335,"model_cost_usd":0.06508},{"model_id":"gemini-3-1-pro","domain":"steuerrecht","score_mean":0.4362169312169312,"score_ci_lo":0.36185185185185187,"score_ci_hi":0.5168518518518519,"top100":44,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.4362169312169312,"hallu_rate_mean":1.1666666666666667,"hallu_prevalence":0.6,"hallu_top100":70,"inter_rater_agreement":0.7015079365079365,"n_judges":4,"tokens_in_total":5420,"tokens_out_total":3505,"model_cost_usd":0.0529},{"model_id":"gemini-3-1-pro","domain":"wirtschaftsrecht","score_mean":0.48667989417989416,"score_ci_lo":0.41099206349206346,"score_ci_hi":0.557579365079365,"top100":49,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.48667989417989416,"hallu_rate_mean":1.7333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":56,"inter_rater_agreement":0.5574074074074074,"n_judges":4,"tokens_in_total":11545,"tokens_out_total":9450,"model_cost_usd":0.13649},{"model_id":"gpt-5","domain":"jura","score_mean":0.6954081632653062,"score_ci_lo":0.615218253968254,"score_ci_hi":0.7623611111111112,"top100":70,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6954081632653062,"hallu_rate_mean":1.0142857142857142,"hallu_prevalence":0.8571428571428571,"hallu_top100":74,"inter_rater_agreement":0.6914399092970521,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":59523,"model_cost_usd":0.952095},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8045386904761905,"score_ci_lo":0.7269345238095238,"score_ci_hi":0.8700644841269842,"top100":80,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8045386904761905,"hallu_rate_mean":0.9125,"hallu_prevalence":0.95,"hallu_top100":77,"inter_rater_agreement":0.825297619047619,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":26950,"model_cost_usd":0.435505},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5509722222222222,"score_ci_lo":0.4165277777777778,"score_ci_hi":0.6801388888888888,"top100":55,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5509722222222222,"hallu_rate_mean":2.25,"hallu_prevalence":0.9333333333333333,"hallu_top100":43,"inter_rater_agreement":0.6471428571428571,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":39885,"model_cost_usd":0.62374},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7917890211640212,"score_ci_lo":0.7355489417989418,"score_ci_hi":0.8471164021164022,"top100":79,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7917890211640212,"hallu_rate_mean":1.275,"hallu_prevalence":0.9666666666666667,"hallu_top100":68,"inter_rater_agreement":0.6768783068783069,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":68255,"model_cost_usd":1.07947},{"model_id":"gpt-5.5-pro","domain":"jura","score_mean":0.7727678571428571,"score_ci_lo":0.7096230158730158,"score_ci_hi":0.8253472222222221,"top100":77,"n_cells":28,"closed_accuracy":null,"open_rubric_score":0.7727678571428571,"hallu_rate_mean":0.5714285714285714,"hallu_prevalence":0.7142857142857143,"hallu_top100":85,"inter_rater_agreement":0.7636904761904761,"n_judges":4,"tokens_in_total":9445,"tokens_out_total":172815,"model_cost_usd":5.231675},{"model_id":"gpt-5.5-pro","domain":"medizin","score_mean":0.8551091269841271,"score_ci_lo":0.8093253968253968,"score_ci_hi":0.8966517857142857,"top100":86,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8551091269841271,"hallu_rate_mean":0.6875,"hallu_prevalence":0.85,"hallu_top100":82,"inter_rater_agreement":0.7329365079365079,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":82847,"model_cost_usd":2.516665},{"model_id":"gpt-5.5-pro","domain":"steuerrecht","score_mean":0.6675347222222222,"score_ci_lo":0.46875,"score_ci_hi":0.8307291666666666,"top100":67,"n_cells":4,"closed_accuracy":null,"open_rubric_score":0.6675347222222222,"hallu_rate_mean":0.9375,"hallu_prevalence":0.75,"hallu_top100":76,"inter_rater_agreement":0.6006944444444444,"n_judges":4,"tokens_in_total":1073,"tokens_out_total":26910,"model_cost_usd":0.812665},{"model_id":"gpt-5.5-pro","domain":"wirtschaftsrecht","score_mean":0.8184672619047619,"score_ci_lo":0.7274255952380952,"score_ci_hi":0.8861011904761904,"top100":82,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8184672619047619,"hallu_rate_mean":0.6625,"hallu_prevalence":0.9,"hallu_top100":83,"inter_rater_agreement":0.7559523809523809,"n_judges":4,"tokens_in_total":6471,"tokens_out_total":119729,"model_cost_usd":3.624225},{"model_id":"grok-4.3","domain":"jura","score_mean":0.4829109977324263,"score_ci_lo":0.42145691609977326,"score_ci_hi":0.5384665532879819,"top100":48,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4829109977324263,"hallu_rate_mean":0.5642857142857143,"hallu_prevalence":0.5428571428571428,"hallu_top100":85,"inter_rater_agreement":0.6688208616780045,"n_judges":4,"tokens_in_total":17561,"tokens_out_total":4534,"model_cost_usd":0.033286},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5640625,"score_ci_lo":0.4887152777777778,"score_ci_hi":0.6275793650793651,"top100":56,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5640625,"hallu_rate_mean":0.4125,"hallu_prevalence":0.55,"hallu_top100":89,"inter_rater_agreement":0.6850198412698413,"n_judges":4,"tokens_in_total":9055,"tokens_out_total":3914,"model_cost_usd":0.021104},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.2730092592592593,"score_ci_lo":0.1786111111111111,"score_ci_hi":0.3723148148148148,"top100":27,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2730092592592593,"hallu_rate_mean":2.0833333333333335,"hallu_prevalence":0.9333333333333333,"hallu_top100":47,"inter_rater_agreement":0.5935449735449736,"n_judges":4,"tokens_in_total":7535,"tokens_out_total":3523,"model_cost_usd":0.018226},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.5128207671957672,"score_ci_lo":0.4494378306878307,"score_ci_hi":0.5734457671957672,"top100":51,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5128207671957672,"hallu_rate_mean":1.2833333333333334,"hallu_prevalence":0.8666666666666667,"hallu_top100":67,"inter_rater_agreement":0.600462962962963,"n_judges":4,"tokens_in_total":16294,"tokens_out_total":8026,"model_cost_usd":0.040432},{"model_id":"kimi-k2.6","domain":"jura","score_mean":0.6131377551020408,"score_ci_lo":0.5457369614512472,"score_ci_hi":0.6810544217687075,"top100":61,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6131377551020408,"hallu_rate_mean":1.4,"hallu_prevalence":0.8857142857142857,"hallu_top100":65,"inter_rater_agreement":0.6696598639455782,"n_judges":4,"tokens_in_total":15188,"tokens_out_total":285468,"model_cost_usd":1.156301},{"model_id":"kimi-k2.6","domain":"medizin","score_mean":0.6390625,"score_ci_lo":0.5487103174603175,"score_ci_hi":0.7118303571428571,"top100":64,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6390625,"hallu_rate_mean":1.1125,"hallu_prevalence":0.9,"hallu_top100":72,"inter_rater_agreement":0.6211309523809524,"n_judges":4,"tokens_in_total":7536,"tokens_out_total":95269,"model_cost_usd":0.388235},{"model_id":"kimi-k2.6","domain":"steuerrecht","score_mean":0.3278174603174603,"score_ci_lo":0.22291666666666668,"score_ci_hi":0.43045634920634923,"top100":33,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3278174603174603,"hallu_rate_mean":3.55,"hallu_prevalence":1,"hallu_top100":11,"inter_rater_agreement":0.5592063492063492,"n_judges":4,"tokens_in_total":6443,"tokens_out_total":148936,"model_cost_usd":0.601865},{"model_id":"kimi-k2.6","domain":"wirtschaftsrecht","score_mean":0.5886871693121694,"score_ci_lo":0.523859126984127,"score_ci_hi":0.65182208994709,"top100":59,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5886871693121694,"hallu_rate_mean":2.7333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":31,"inter_rater_agreement":0.5682407407407407,"n_judges":4,"tokens_in_total":14354,"tokens_out_total":247838,"model_cost_usd":1.004988},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4850198412698412,"score_ci_lo":0.4023611111111111,"score_ci_hi":0.5553373015873015,"top100":49,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4850198412698412,"hallu_rate_mean":1.7,"hallu_prevalence":0.8285714285714286,"hallu_top100":57,"inter_rater_agreement":0.6413605442176871,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10447,"model_cost_usd":0.089346},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.7126984126984126,"score_ci_lo":0.6493055555555556,"score_ci_hi":0.7775297619047619,"top100":71,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7126984126984126,"hallu_rate_mean":1.025,"hallu_prevalence":0.95,"hallu_top100":74,"inter_rater_agreement":0.6989087301587301,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10165,"model_cost_usd":0.074328},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.2460780423280423,"score_ci_lo":0.16484126984126984,"score_ci_hi":0.33607804232804234,"top100":25,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.2460780423280423,"hallu_rate_mean":3.5,"hallu_prevalence":1,"hallu_top100":12,"inter_rater_agreement":0.5046031746031746,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":6928,"model_cost_usd":0.052712},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5604166666666667,"score_ci_lo":0.49665674603174603,"score_ci_hi":0.6227777777777778,"top100":56,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5604166666666667,"hallu_rate_mean":2.45,"hallu_prevalence":0.9333333333333333,"hallu_top100":38,"inter_rater_agreement":0.5235714285714286,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":18886,"model_cost_usd":0.137056}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7762361111111111,"score_ci_lo":0.7480327380952382,"score_ci_hi":0.8038829365079365,"top100":78,"n_cells":100,"hallu_rate_mean_across_domains":1.63,"hallu_top100_across_domains":59},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.744406746031746,"score_ci_lo":0.7115396825396826,"score_ci_hi":0.7750873015873015,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.16,"hallu_top100_across_domains":71},{"model_id":"gemini-3-1-pro","score_mean_across_domains":0.5392281746031746,"score_ci_lo":0.4996438492063492,"score_ci_hi":0.5762063492063492,"top100":54,"n_cells":100,"hallu_rate_mean_across_domains":1.0375,"hallu_top100_across_domains":74},{"model_id":"gpt-5","score_mean_across_domains":0.7244831349206349,"score_ci_lo":0.6796031746031745,"score_ci_hi":0.765266865079365,"top100":72,"n_cells":100,"hallu_rate_mean_across_domains":1.2575,"hallu_top100_across_domains":68},{"model_id":"gpt-5.5-pro","score_mean_across_domains":0.802488425925926,"score_ci_lo":0.7646963183421517,"score_ci_hi":0.8365423831569665,"top100":80,"n_cells":72,"hallu_rate_mean_across_domains":0.6493055555555556,"hallu_top100_across_domains":83},{"model_id":"grok-4.3","score_mean_across_domains":0.4766289682539683,"score_ci_lo":0.43680753968253966,"score_ci_hi":0.5147886904761905,"top100":48,"n_cells":100,"hallu_rate_mean_across_domains":0.9775,"hallu_top100_across_domains":75},{"model_id":"kimi-k2.6","score_mean_across_domains":0.5681894841269841,"score_ci_lo":0.527639880952381,"score_ci_hi":0.6114573412698412,"top100":57,"n_cells":100,"hallu_rate_mean_across_domains":2.065,"hallu_top100_across_domains":48},{"model_id":"mistral-large-2","score_mean_across_domains":0.5173333333333333,"score_ci_lo":0.46645535714285713,"score_ci_hi":0.5639355158730158,"top100":52,"n_cells":100,"hallu_rate_mean_across_domains":2.06,"hallu_top100_across_domains":48}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8582,"oracle_top100":86,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7871,"best_single_model_top100":79},{"domain":"medizin","n_items":20,"oracle_mean":0.9195,"oracle_top100":92,"best_single_model_id":"gpt-5.5-pro","best_single_model_mean":0.8551,"best_single_model_top100":86},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7577,"oracle_top100":76,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.712,"best_single_model_top100":71},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8574,"oracle_top100":86,"best_single_model_id":"gpt-5.5-pro","best_single_model_mean":0.8185,"best_single_model_top100":82}],"cost":{"model_calls_usd":25.5477,"judge_calls_usd":80.4661,"total_usd":106.0137,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":2134166,"tokens_out_total":471145,"cost_usd":22.449455},{"judge_id":"claude-opus-4-8","tokens_in_total":2130306,"tokens_out_total":592290,"cost_usd":25.45878},{"judge_id":"gpt-5","tokens_in_total":1101448,"tokens_out_total":1440549,"cost_usd":27.115475},{"judge_id":"mistral-large-2","tokens_in_total":1181553,"tokens_out_total":513206,"cost_usd":5.442342}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":1272.16,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260619T195640-4ce8c8/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260619T195640-4ce8c8/progress.jsonl","cost_total_usd":106.0137}