{"run_id":"run-20260904T151155-c623e0","started_at":1788534715,"summary":{"run_id":"run-20260904T151155-c623e0","question_bank_version":"2026-05-17-v1","code_version":"0.2.7","started_at":1788534715.097595,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-fable-5-1","domain":"jura","score_mean":0.8760515873015873,"score_ci_lo":0.830297619047619,"score_ci_hi":0.918968253968254,"top100":88,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8760515873015873,"hallu_rate_mean":1.4857142857142858,"hallu_prevalence":0.9428571428571428,"hallu_top100":62,"inter_rater_agreement":0.8556802721088435,"n_judges":4,"tokens_in_total":22845,"tokens_out_total":104804,"model_cost_usd":5.46865},{"model_id":"claude-fable-5-1","domain":"medizin","score_mean":0.8536706349206349,"score_ci_lo":0.7661485890652557,"score_ci_hi":0.9192846119929453,"top100":85,"n_cells":18,"closed_accuracy":null,"open_rubric_score":0.8536706349206349,"hallu_rate_mean":1.3333333333333333,"hallu_prevalence":1,"hallu_top100":66,"inter_rater_agreement":0.8421516754850088,"n_judges":4,"tokens_in_total":10153,"tokens_out_total":39428,"model_cost_usd":2.07293},{"model_id":"claude-fable-5-1","domain":"steuerrecht","score_mean":0.7637103174603175,"score_ci_lo":0.6431283068783069,"score_ci_hi":0.8568055555555555,"top100":76,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7637103174603175,"hallu_rate_mean":3.683333333333333,"hallu_prevalence":1,"hallu_top100":7,"inter_rater_agreement":0.7451322751322752,"n_judges":4,"tokens_in_total":9765,"tokens_out_total":97866,"model_cost_usd":4.99095},{"model_id":"claude-fable-5-1","domain":"wirtschaftsrecht","score_mean":0.739103835978836,"score_ci_lo":0.6578505291005291,"score_ci_hi":0.8122982804232804,"top100":74,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.739103835978836,"hallu_rate_mean":3.45,"hallu_prevalence":1,"hallu_top100":13,"inter_rater_agreement":0.7195767195767196,"n_judges":4,"tokens_in_total":22172,"tokens_out_total":143424,"model_cost_usd":7.39292},{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8026348039215686,"score_ci_lo":0.7347834967320261,"score_ci_hi":0.8669117647058824,"top100":80,"n_cells":34,"closed_accuracy":null,"open_rubric_score":0.8026348039215686,"hallu_rate_mean":0.8676470588235294,"hallu_prevalence":0.7941176470588235,"hallu_top100":78,"inter_rater_agreement":0.8209617180205415,"n_judges":4,"tokens_in_total":22343,"tokens_out_total":28479,"model_cost_usd":0.82369},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8310019841269842,"score_ci_lo":0.7586557539682539,"score_ci_hi":0.8944196428571429,"top100":83,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8310019841269842,"hallu_rate_mean":0.9875,"hallu_prevalence":0.95,"hallu_top100":75,"inter_rater_agreement":0.7178571428571429,"n_judges":4,"tokens_in_total":11495,"tokens_out_total":21383,"model_cost_usd":0.59205},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7337433862433862,"score_ci_lo":0.6471693121693122,"score_ci_hi":0.8170833333333333,"top100":73,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7337433862433862,"hallu_rate_mean":2.2333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":44,"inter_rater_agreement":0.6513227513227513,"n_judges":4,"tokens_in_total":9810,"tokens_out_total":20146,"model_cost_usd":0.5527},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7780324074074074,"score_ci_lo":0.7337962962962963,"score_ci_hi":0.8266236772486771,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7780324074074074,"hallu_rate_mean":2.5,"hallu_prevalence":1,"hallu_top100":37,"inter_rater_agreement":0.7202248677248677,"n_judges":4,"tokens_in_total":22262,"tokens_out_total":41663,"model_cost_usd":1.152885},{"model_id":"claude-opus-4-8","domain":"jura","score_mean":0.8003146258503401,"score_ci_lo":0.7569189342403628,"score_ci_hi":0.8416269841269841,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8003146258503401,"hallu_rate_mean":0.85,"hallu_prevalence":0.8,"hallu_top100":78,"inter_rater_agreement":0.7763265306122449,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":30917,"model_cost_usd":0.8868},{"model_id":"claude-opus-4-8","domain":"medizin","score_mean":0.8309027777777779,"score_ci_lo":0.7708333333333333,"score_ci_hi":0.8838541666666666,"top100":83,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8309027777777779,"hallu_rate_mean":0.85,"hallu_prevalence":1,"hallu_top100":78,"inter_rater_agreement":0.7805555555555556,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":21291,"model_cost_usd":0.58925},{"model_id":"claude-opus-4-8","domain":"steuerrecht","score_mean":0.696137566137566,"score_ci_lo":0.6078042328042328,"score_ci_hi":0.7721759259259259,"top100":70,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.696137566137566,"hallu_rate_mean":1.6166666666666667,"hallu_prevalence":1,"hallu_top100":59,"inter_rater_agreement":0.7749470899470899,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":20696,"model_cost_usd":0.566075},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","score_mean":0.6929464285714285,"score_ci_lo":0.635327380952381,"score_ci_hi":0.7527579365079365,"top100":69,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6929464285714285,"hallu_rate_mean":1.8833333333333333,"hallu_prevalence":0.9666666666666667,"hallu_top100":52,"inter_rater_agreement":0.6917857142857143,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":45247,"model_cost_usd":1.241735},{"model_id":"claude-opus-5","domain":"jura","score_mean":0.905187074829932,"score_ci_lo":0.8693027210884353,"score_ci_hi":0.9361904761904761,"top100":91,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.905187074829932,"hallu_rate_mean":1.6857142857142857,"hallu_prevalence":0.9428571428571428,"hallu_top100":57,"inter_rater_agreement":0.8367233560090703,"n_judges":4,"tokens_in_total":22775,"tokens_out_total":126541,"model_cost_usd":3.2774},{"model_id":"claude-opus-5","domain":"medizin","score_mean":0.9070188492063492,"score_ci_lo":0.8495535714285714,"score_ci_hi":0.9584573412698413,"top100":91,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.9070188492063492,"hallu_rate_mean":1.45,"hallu_prevalence":1,"hallu_top100":63,"inter_rater_agreement":0.8933531746031745,"n_judges":4,"tokens_in_total":11395,"tokens_out_total":64191,"model_cost_usd":1.66175},{"model_id":"claude-opus-5","domain":"steuerrecht","score_mean":0.8546296296296296,"score_ci_lo":0.7620833333333333,"score_ci_hi":0.9396759259259259,"top100":85,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.8546296296296296,"hallu_rate_mean":2.966666666666667,"hallu_prevalence":1,"hallu_top100":25,"inter_rater_agreement":0.7975925925925925,"n_judges":4,"tokens_in_total":9735,"tokens_out_total":115929,"model_cost_usd":2.9469},{"model_id":"claude-opus-5","domain":"wirtschaftsrecht","score_mean":0.7873941798941799,"score_ci_lo":0.7195469576719576,"score_ci_hi":0.8582473544973545,"top100":79,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7873941798941799,"hallu_rate_mean":2.5166666666666666,"hallu_prevalence":1,"hallu_top100":37,"inter_rater_agreement":0.6527380952380952,"n_judges":4,"tokens_in_total":22112,"tokens_out_total":224820,"model_cost_usd":5.73106},{"model_id":"deepseek-v4-pro","domain":"jura","score_mean":0.6130243764172335,"score_ci_lo":0.5381207482993198,"score_ci_hi":0.6796201814058956,"top100":61,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6130243764172335,"hallu_rate_mean":0.7857142857142857,"hallu_prevalence":0.6571428571428571,"hallu_top100":80,"inter_rater_agreement":0.6778231292517007,"n_judges":4,"tokens_in_total":16523,"tokens_out_total":228064,"model_cost_usd":0.205603},{"model_id":"deepseek-v4-pro","domain":"medizin","score_mean":0.6998511904761905,"score_ci_lo":0.6251984126984127,"score_ci_hi":0.7766369047619047,"top100":70,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6998511904761905,"hallu_rate_mean":0.3875,"hallu_prevalence":0.65,"hallu_top100":90,"inter_rater_agreement":0.6645833333333333,"n_judges":4,"tokens_in_total":8383,"tokens_out_total":45676,"model_cost_usd":0.043385},{"model_id":"deepseek-v4-pro","domain":"steuerrecht","score_mean":0.4129426129426129,"score_ci_lo":0.28826312576312574,"score_ci_hi":0.5367139804639804,"top100":41,"n_cells":13,"closed_accuracy":null,"open_rubric_score":0.4129426129426129,"hallu_rate_mean":2.7884615384615383,"hallu_prevalence":0.9230769230769231,"hallu_top100":30,"inter_rater_agreement":0.5588217338217338,"n_judges":4,"tokens_in_total":6033,"tokens_out_total":118383,"model_cost_usd":0.105618},{"model_id":"deepseek-v4-pro","domain":"wirtschaftsrecht","score_mean":0.5651818783068784,"score_ci_lo":0.494646164021164,"score_ci_hi":0.6230257936507936,"top100":57,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5651818783068784,"hallu_rate_mean":1.3333333333333333,"hallu_prevalence":0.9,"hallu_top100":66,"inter_rater_agreement":0.6376190476190476,"n_judges":4,"tokens_in_total":15333,"tokens_out_total":217183,"model_cost_usd":0.195619},{"model_id":"gemini-3-1-pro","domain":"jura","score_mean":0.589562908496732,"score_ci_lo":0.5186478758169935,"score_ci_hi":0.6494281045751634,"top100":59,"n_cells":34,"closed_accuracy":null,"open_rubric_score":0.589562908496732,"hallu_rate_mean":0.7058823529411765,"hallu_prevalence":0.6764705882352942,"hallu_top100":82,"inter_rater_agreement":0.6784780578898225,"n_judges":4,"tokens_in_total":12520,"tokens_out_total":6302,"model_cost_usd":0.100664},{"model_id":"gemini-3-1-pro","domain":"medizin","score_mean":0.6114087301587301,"score_ci_lo":0.5394097222222223,"score_ci_hi":0.6779265873015874,"top100":61,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6114087301587301,"hallu_rate_mean":0.35,"hallu_prevalence":0.5,"hallu_top100":91,"inter_rater_agreement":0.7271825396825398,"n_judges":4,"tokens_in_total":6530,"tokens_out_total":4481,"model_cost_usd":0.066832},{"model_id":"gemini-3-1-pro","domain":"steuerrecht","score_mean":0.4556051587301587,"score_ci_lo":0.3675595238095238,"score_ci_hi":0.5487103174603175,"top100":46,"n_cells":14,"closed_accuracy":null,"open_rubric_score":0.4556051587301587,"hallu_rate_mean":1.6964285714285714,"hallu_prevalence":0.8571428571428571,"hallu_top100":57,"inter_rater_agreement":0.6040816326530613,"n_judges":4,"tokens_in_total":5162,"tokens_out_total":3464,"model_cost_usd":0.051892},{"model_id":"gemini-3-1-pro","domain":"wirtschaftsrecht","score_mean":0.48861607142857144,"score_ci_lo":0.41855867346938774,"score_ci_hi":0.552923044217687,"top100":49,"n_cells":28,"closed_accuracy":null,"open_rubric_score":0.48861607142857144,"hallu_rate_mean":1.5982142857142858,"hallu_prevalence":1,"hallu_top100":60,"inter_rater_agreement":0.570124716553288,"n_judges":4,"tokens_in_total":10845,"tokens_out_total":8595,"model_cost_usd":0.12483},{"model_id":"glm-5.3","domain":"jura","score_mean":0.7386970899470899,"score_ci_lo":0.6364219576719576,"score_ci_hi":0.8191137566137566,"top100":74,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7386970899470899,"hallu_rate_mean":2.05,"hallu_prevalence":0.9666666666666667,"hallu_top100":48,"inter_rater_agreement":0.7633994708994708,"n_judges":4,"tokens_in_total":11176,"tokens_out_total":256332,"model_cost_usd":0.570636},{"model_id":"glm-5.3","domain":"medizin","score_mean":0.8385168650793651,"score_ci_lo":0.767906746031746,"score_ci_hi":0.8968501984126984,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8385168650793651,"hallu_rate_mean":1.3375,"hallu_prevalence":1,"hallu_top100":66,"inter_rater_agreement":0.8211309523809524,"n_judges":4,"tokens_in_total":6938,"tokens_out_total":98707,"model_cost_usd":0.221318},{"model_id":"glm-5.3","domain":"steuerrecht","score_mean":0.7597222222222223,"score_ci_lo":0.6666666666666666,"score_ci_hi":0.8125,"top100":76,"n_cells":3,"closed_accuracy":null,"open_rubric_score":0.7597222222222223,"hallu_rate_mean":3.1666666666666665,"hallu_prevalence":1,"hallu_top100":20,"inter_rater_agreement":0.7805555555555556,"n_judges":4,"tokens_in_total":924,"tokens_out_total":24221,"model_cost_usd":0.053841},{"model_id":"glm-5.3","domain":"wirtschaftsrecht","score_mean":0.7217724867724867,"score_ci_lo":0.5861772486772487,"score_ci_hi":0.8406878306878307,"top100":72,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7217724867724867,"hallu_rate_mean":3.0833333333333335,"hallu_prevalence":1,"hallu_top100":22,"inter_rater_agreement":0.7175396825396825,"n_judges":4,"tokens_in_total":5895,"tokens_out_total":178968,"model_cost_usd":0.397267},{"model_id":"gpt-5","domain":"jura","score_mean":0.7154336734693877,"score_ci_lo":0.638109410430839,"score_ci_hi":0.7814682539682539,"top100":72,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7154336734693877,"hallu_rate_mean":1.2,"hallu_prevalence":0.8,"hallu_top100":70,"inter_rater_agreement":0.7181859410430839,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":57439,"model_cost_usd":0.920835},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8406994047619047,"score_ci_lo":0.7817956349206349,"score_ci_hi":0.8951140873015874,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8406994047619047,"hallu_rate_mean":0.7875,"hallu_prevalence":0.95,"hallu_top100":80,"inter_rater_agreement":0.8181547619047619,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":26842,"model_cost_usd":0.433885},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.5863359788359789,"score_ci_lo":0.4300859788359788,"score_ci_hi":0.7194907407407407,"top100":59,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5863359788359789,"hallu_rate_mean":2.466666666666667,"hallu_prevalence":1,"hallu_top100":38,"inter_rater_agreement":0.5976190476190476,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":41970,"model_cost_usd":0.655015},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7709060846560847,"score_ci_lo":0.7200462962962962,"score_ci_hi":0.8257936507936507,"top100":77,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7709060846560847,"hallu_rate_mean":1.2583333333333333,"hallu_prevalence":0.9333333333333333,"hallu_top100":68,"inter_rater_agreement":0.6296428571428571,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":68597,"model_cost_usd":1.0846},{"model_id":"gpt-5.6-sol","domain":"jura","score_mean":0.7593707482993197,"score_ci_lo":0.7012755102040816,"score_ci_hi":0.8158134920634921,"top100":76,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7593707482993197,"hallu_rate_mean":0.9428571428571428,"hallu_prevalence":0.8285714285714286,"hallu_top100":76,"inter_rater_agreement":0.7212018140589569,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":135598,"model_cost_usd":4.12719},{"model_id":"gpt-5.6-sol","domain":"medizin","score_mean":0.8201884920634921,"score_ci_lo":0.766889880952381,"score_ci_hi":0.8741071428571429,"top100":82,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8201884920634921,"hallu_rate_mean":0.7125,"hallu_prevalence":0.85,"hallu_top100":82,"inter_rater_agreement":0.6898809523809524,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":32597,"model_cost_usd":1.009165},{"model_id":"gpt-5.6-sol","domain":"steuerrecht","score_mean":0.6127777777777778,"score_ci_lo":0.52625,"score_ci_hi":0.7066666666666667,"top100":61,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6127777777777778,"hallu_rate_mean":2.7,"hallu_prevalence":1,"hallu_top100":32,"inter_rater_agreement":0.5317989417989418,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":121699,"model_cost_usd":3.676435},{"model_id":"gpt-5.6-sol","domain":"wirtschaftsrecht","score_mean":0.778125,"score_ci_lo":0.7264384920634921,"score_ci_hi":0.8287698412698412,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.778125,"hallu_rate_mean":1.375,"hallu_prevalence":1,"hallu_top100":65,"inter_rater_agreement":0.6963888888888888,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":189898,"model_cost_usd":5.752585},{"model_id":"gpt-6-astra","domain":"jura","score_mean":0.7065873015873015,"score_ci_lo":0.6366269841269842,"score_ci_hi":0.768313492063492,"top100":71,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7065873015873015,"hallu_rate_mean":1.3071428571428572,"hallu_prevalence":0.9428571428571428,"hallu_top100":67,"inter_rater_agreement":0.7030725623582766,"n_judges":4,"tokens_in_total":11850,"tokens_out_total":112111,"model_cost_usd":5.72405},{"model_id":"gpt-6-astra","domain":"medizin","score_mean":0.8437996031746031,"score_ci_lo":0.8069940476190476,"score_ci_hi":0.8824652777777778,"top100":84,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8437996031746031,"hallu_rate_mean":0.9375,"hallu_prevalence":0.95,"hallu_top100":76,"inter_rater_agreement":0.781547619047619,"n_judges":4,"tokens_in_total":6251,"tokens_out_total":38383,"model_cost_usd":1.98166},{"model_id":"gpt-6-astra","domain":"steuerrecht","score_mean":0.6843849206349206,"score_ci_lo":0.578968253968254,"score_ci_hi":0.7693121693121693,"top100":68,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6843849206349206,"hallu_rate_mean":2.3,"hallu_prevalence":1,"hallu_top100":42,"inter_rater_agreement":0.736984126984127,"n_judges":4,"tokens_in_total":5093,"tokens_out_total":86893,"model_cost_usd":4.39558},{"model_id":"gpt-6-astra","domain":"wirtschaftsrecht","score_mean":0.7624140211640211,"score_ci_lo":0.706223544973545,"score_ci_hi":0.8152579365079365,"top100":76,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7624140211640211,"hallu_rate_mean":1.5166666666666666,"hallu_prevalence":1,"hallu_top100":62,"inter_rater_agreement":0.6670899470899471,"n_judges":4,"tokens_in_total":11129,"tokens_out_total":157203,"model_cost_usd":7.97144},{"model_id":"grok-4.5","domain":"jura","score_mean":0.6725226757369615,"score_ci_lo":0.6055357142857143,"score_ci_hi":0.7247052154195012,"top100":67,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6725226757369615,"hallu_rate_mean":0.42857142857142855,"hallu_prevalence":0.5142857142857142,"hallu_top100":89,"inter_rater_agreement":0.713344671201814,"n_judges":4,"tokens_in_total":28906,"tokens_out_total":6885,"model_cost_usd":0.099122},{"model_id":"grok-4.5","domain":"medizin","score_mean":0.6746775793650793,"score_ci_lo":0.5940724206349206,"score_ci_hi":0.7525793650793651,"top100":67,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6746775793650793,"hallu_rate_mean":0.4375,"hallu_prevalence":0.6,"hallu_top100":89,"inter_rater_agreement":0.7137896825396826,"n_judges":4,"tokens_in_total":15932,"tokens_out_total":5355,"model_cost_usd":0.063994},{"model_id":"grok-4.5","domain":"steuerrecht","score_mean":0.5037367724867725,"score_ci_lo":0.41800925925925925,"score_ci_hi":0.5962830687830688,"top100":50,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5037367724867725,"hallu_rate_mean":1.2333333333333334,"hallu_prevalence":0.8666666666666667,"hallu_top100":69,"inter_rater_agreement":0.5926190476190476,"n_judges":4,"tokens_in_total":12303,"tokens_out_total":5227,"model_cost_usd":0.055968},{"model_id":"grok-4.5","domain":"wirtschaftsrecht","score_mean":0.6636937830687831,"score_ci_lo":0.6102876984126984,"score_ci_hi":0.7159656084656085,"top100":66,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6636937830687831,"hallu_rate_mean":0.7916666666666666,"hallu_prevalence":0.7333333333333333,"hallu_top100":80,"inter_rater_agreement":0.6081481481481481,"n_judges":4,"tokens_in_total":25515,"tokens_out_total":14317,"model_cost_usd":0.136932},{"model_id":"grok-4.6","domain":"jura","score_mean":0.6835912698412698,"score_ci_lo":0.621626984126984,"score_ci_hi":0.7357539682539683,"top100":68,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6835912698412698,"hallu_rate_mean":0.38571428571428573,"hallu_prevalence":0.42857142857142855,"hallu_top100":90,"inter_rater_agreement":0.7236961451247166,"n_judges":4,"tokens_in_total":33876,"tokens_out_total":7034,"model_cost_usd":0.109956},{"model_id":"grok-4.6","domain":"medizin","score_mean":0.6807539682539683,"score_ci_lo":0.6128720238095238,"score_ci_hi":0.7503968253968254,"top100":68,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6807539682539683,"hallu_rate_mean":0.4375,"hallu_prevalence":0.65,"hallu_top100":89,"inter_rater_agreement":0.6843253968253968,"n_judges":4,"tokens_in_total":18772,"tokens_out_total":4841,"model_cost_usd":0.06659},{"model_id":"grok-4.6","domain":"steuerrecht","score_mean":0.5387169312169312,"score_ci_lo":0.46547619047619043,"score_ci_hi":0.6160714285714286,"top100":54,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5387169312169312,"hallu_rate_mean":1.8333333333333333,"hallu_prevalence":0.9333333333333333,"hallu_top100":54,"inter_rater_agreement":0.5903968253968254,"n_judges":4,"tokens_in_total":14433,"tokens_out_total":6096,"model_cost_usd":0.065442},{"model_id":"grok-4.6","domain":"wirtschaftsrecht","score_mean":0.6513789682539682,"score_ci_lo":0.5932936507936508,"score_ci_hi":0.7101884920634921,"top100":65,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6513789682539682,"hallu_rate_mean":1.2333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":69,"inter_rater_agreement":0.6504232804232805,"n_judges":4,"tokens_in_total":29775,"tokens_out_total":14996,"model_cost_usd":0.149526},{"model_id":"kimi-k3","domain":"jura","score_mean":0.7969450280112045,"score_ci_lo":0.7200659430438842,"score_ci_hi":0.8572799953314659,"top100":80,"n_cells":34,"closed_accuracy":null,"open_rubric_score":0.7969450280112045,"hallu_rate_mean":1.625,"hallu_prevalence":0.8823529411764706,"hallu_top100":59,"inter_rater_agreement":0.7927754435107376,"n_judges":4,"tokens_in_total":17369,"tokens_out_total":189489,"model_cost_usd":0.774457},{"model_id":"kimi-k3","domain":"medizin","score_mean":0.8324652777777779,"score_ci_lo":0.7729166666666667,"score_ci_hi":0.8970486111111111,"top100":83,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8324652777777779,"hallu_rate_mean":1.3,"hallu_prevalence":0.9,"hallu_top100":67,"inter_rater_agreement":0.7402777777777778,"n_judges":4,"tokens_in_total":9096,"tokens_out_total":69993,"model_cost_usd":0.288613},{"model_id":"kimi-k3","domain":"steuerrecht","score_mean":0.5646825396825397,"score_ci_lo":0.33789682539682536,"score_ci_hi":0.7579365079365079,"top100":56,"n_cells":7,"closed_accuracy":null,"open_rubric_score":0.5646825396825397,"hallu_rate_mean":3.0357142857142856,"hallu_prevalence":1,"hallu_top100":24,"inter_rater_agreement":0.7071428571428572,"n_judges":4,"tokens_in_total":3255,"tokens_out_total":61236,"model_cost_usd":0.248036},{"model_id":"kimi-k3","domain":"wirtschaftsrecht","score_mean":0.7791815476190476,"score_ci_lo":0.6758333333333333,"score_ci_hi":0.8641517857142856,"top100":78,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.7791815476190476,"hallu_rate_mean":2.9375,"hallu_prevalence":1,"hallu_top100":26,"inter_rater_agreement":0.697202380952381,"n_judges":4,"tokens_in_total":10409,"tokens_out_total":141047,"model_cost_usd":0.574077},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4594756235827664,"score_ci_lo":0.38116213151927436,"score_ci_hi":0.5368565759637188,"top100":46,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4594756235827664,"hallu_rate_mean":1.8857142857142857,"hallu_prevalence":0.8285714285714286,"hallu_top100":52,"inter_rater_agreement":0.6699319727891156,"n_judges":4,"tokens_in_total":13332,"tokens_out_total":10392,"model_cost_usd":0.089016},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6488343253968254,"score_ci_lo":0.6016121031746031,"score_ci_hi":0.6983878968253968,"top100":65,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6488343253968254,"hallu_rate_mean":1.425,"hallu_prevalence":0.95,"hallu_top100":64,"inter_rater_agreement":0.6636904761904762,"n_judges":4,"tokens_in_total":6669,"tokens_out_total":10087,"model_cost_usd":0.07386},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.28414021164021164,"score_ci_lo":0.18515873015873016,"score_ci_hi":0.38819444444444445,"top100":28,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.28414021164021164,"hallu_rate_mean":3.45,"hallu_prevalence":0.9333333333333333,"hallu_top100":13,"inter_rater_agreement":0.5542328042328042,"n_judges":4,"tokens_in_total":5572,"tokens_out_total":6959,"model_cost_usd":0.052898},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5567493386243386,"score_ci_lo":0.483614417989418,"score_ci_hi":0.6320039682539683,"top100":56,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5567493386243386,"hallu_rate_mean":2.15,"hallu_prevalence":0.9666666666666667,"hallu_top100":46,"inter_rater_agreement":0.5599206349206349,"n_judges":4,"tokens_in_total":11870,"tokens_out_total":17947,"model_cost_usd":0.131422}],"by_model_overall":[{"model_id":"claude-fable-5-1","score_mean_across_domains":0.8128229267897634,"score_ci_lo":0.7747479348882409,"score_ci_hi":0.8505749919015225,"top100":81,"n_cells":98,"hallu_rate_mean_across_domains":2.395408163265306,"hallu_top100_across_domains":40},{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7904721821388488,"score_ci_lo":0.754930254930255,"score_ci_hi":0.8238706509539844,"top100":79,"n_cells":99,"hallu_rate_mean_across_domains":1.5934343434343434,"hallu_top100_across_domains":60},{"model_id":"claude-opus-4-8","score_mean_across_domains":0.7585952380952381,"score_ci_lo":0.7273045634920635,"score_ci_hi":0.7871795634920634,"top100":76,"n_cells":100,"hallu_rate_mean_across_domains":1.275,"hallu_top100_across_domains":68},{"model_id":"claude-opus-5","score_mean_across_domains":0.8626319444444444,"score_ci_lo":0.8313948412698413,"score_ci_hi":0.8911547619047618,"top100":86,"n_cells":100,"hallu_rate_mean_across_domains":2.08,"hallu_top100_across_domains":48},{"model_id":"deepseek-v4-pro","score_mean_across_domains":0.589557013281503,"score_ci_lo":0.5454921849692258,"score_ci_hi":0.6301222870100421,"top100":59,"n_cells":98,"hallu_rate_mean_across_domains":1.1377551020408163,"hallu_top100_across_domains":71},{"model_id":"gemini-3-1-pro","score_mean_across_domains":0.5451357886904762,"score_ci_lo":0.5057426008597884,"score_ci_hi":0.5810464203042328,"top100":55,"n_cells":96,"hallu_rate_mean_across_domains":1.0364583333333333,"hallu_top100_across_domains":74},{"model_id":"glm-5.3","score_mean_across_domains":0.765250058356676,"score_ci_lo":0.7110512955182073,"score_ci_hi":0.8186128618113911,"top100":77,"n_cells":68,"hallu_rate_mean_across_domains":2.1176470588235294,"hallu_top100_across_domains":47},{"model_id":"gpt-5","score_mean_across_domains":0.7377638888888889,"score_ci_lo":0.6916190476190477,"score_ci_hi":0.7789226190476191,"top100":74,"n_cells":100,"hallu_rate_mean_across_domains":1.325,"hallu_top100_across_domains":66},{"model_id":"gpt-5.6-sol","score_mean_across_domains":0.7551716269841271,"score_ci_lo":0.7221994047619047,"score_ci_hi":0.7899811507936508,"top100":76,"n_cells":100,"hallu_rate_mean_across_domains":1.29,"hallu_top100_across_domains":67},{"model_id":"gpt-6-astra","score_mean_across_domains":0.7474474206349206,"score_ci_lo":0.7110873015873015,"score_ci_hi":0.779735119047619,"top100":75,"n_cells":100,"hallu_rate_mean_across_domains":1.445,"hallu_top100_across_domains":63},{"model_id":"grok-4.5","score_mean_across_domains":0.6449871031746032,"score_ci_lo":0.6087718253968254,"score_ci_hi":0.6804642857142856,"top100":64,"n_cells":100,"hallu_rate_mean_across_domains":0.66,"hallu_top100_across_domains":83},{"model_id":"grok-4.6","score_mean_across_domains":0.6516289682539683,"score_ci_lo":0.6182261904761905,"score_ci_hi":0.6855744047619048,"top100":65,"n_cells":100,"hallu_rate_mean_across_domains":0.8675,"hallu_top100_across_domains":78},{"model_id":"kimi-k3","score_mean_across_domains":0.7812573486184597,"score_ci_lo":0.7350627082108564,"score_ci_hi":0.8255768665490888,"top100":78,"n_cells":81,"hallu_rate_mean_across_domains":1.9907407407407407,"hallu_top100_across_domains":50},{"model_id":"mistral-large-2","score_mean_across_domains":0.5002291666666667,"score_ci_lo":0.45065178571428566,"score_ci_hi":0.5465853174603175,"top100":50,"n_cells":100,"hallu_rate_mean_across_domains":2.1075,"hallu_top100_across_domains":47}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.9433,"oracle_top100":94,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.9052,"best_single_model_top100":91},{"domain":"medizin","n_items":20,"oracle_mean":0.9726,"oracle_top100":97,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.907,"best_single_model_top100":91},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.9213,"oracle_top100":92,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.8546,"best_single_model_top100":85},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.9101,"oracle_top100":91,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.7874,"best_single_model_top100":79}],"cost":{"model_calls_usd":82.7976,"judge_calls_usd":150.6431,"total_usd":233.4407,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":4176748,"tokens_out_total":851320,"cost_usd":42.16674},{"judge_id":"claude-opus-4-8","tokens_in_total":4170048,"tokens_out_total":1024758,"cost_usd":46.46919},{"judge_id":"gpt-5","tokens_in_total":2147692,"tokens_out_total":2733161,"cost_usd":51.735875},{"judge_id":"mistral-large-2","tokens_in_total":2306996,"tokens_out_total":942884,"cost_usd":10.271296}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":2416.68,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[],"merges":[{"run_id":"run-20260907T150129-3c2503","models":["gpt-6-astra"],"total_usd":32.051,"merged_at":"2026-09-07T16:19:54Z"}]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260904T151155-c623e0/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260904T151155-c623e0/progress.jsonl","cost_total_usd":233.4407}