{"run_id":"run-20260528T202802-6ce096","started_at":1780000082,"summary":{"run_id":"run-20260528T202802-6ce096","question_bank_version":"2026-05-17-v1","code_version":"0.2.3","started_at":1780000082.8041415,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8243348450491307,"score_ci_lo":0.7883786848072563,"score_ci_hi":0.8625925925925926,"top100":82,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.8243348450491307,"hallu_rate_mean":0.8571428571428571,"hallu_prevalence":0.7142857142857143,"hallu_top100":78,"inter_rater_agreement":0.8313151927437641,"n_judges":3,"tokens_in_total":22950,"tokens_out_total":30287,"model_cost_usd":2.615775},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8677910052910052,"score_ci_lo":0.8175595238095237,"score_ci_hi":0.9135582010582011,"top100":87,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.8677910052910052,"hallu_rate_mean":1.4,"hallu_prevalence":0.9,"hallu_top100":65,"inter_rater_agreement":0.8463293650793651,"n_judges":3,"tokens_in_total":11495,"tokens_out_total":20568,"model_cost_usd":1.715025},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7637213403880071,"score_ci_lo":0.6908201058201058,"score_ci_hi":0.8334126984126985,"top100":76,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.7637213403880071,"hallu_rate_mean":2.3777777777777778,"hallu_prevalence":0.9333333333333333,"hallu_top100":40,"inter_rater_agreement":0.808068783068783,"n_judges":3,"tokens_in_total":9810,"tokens_out_total":18349,"model_cost_usd":1.523325},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7243959435626103,"score_ci_lo":0.6672222222222222,"score_ci_hi":0.7785405643738977,"top100":72,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7243959435626103,"hallu_rate_mean":2.566666666666667,"hallu_prevalence":1,"hallu_top100":35,"inter_rater_agreement":0.6798941798941799,"n_judges":3,"tokens_in_total":22262,"tokens_out_total":42187,"model_cost_usd":3.497955},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.6083144368858655,"score_ci_lo":0.5485789871504156,"score_ci_hi":0.663817082388511,"top100":61,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6083144368858655,"hallu_rate_mean":0.45714285714285713,"hallu_prevalence":0.37142857142857144,"hallu_top100":88,"inter_rater_agreement":0.730453514739229,"n_judges":3,"tokens_in_total":12898,"tokens_out_total":6845,"model_cost_usd":0.084573},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.564781746031746,"score_ci_lo":0.4791666666666667,"score_ci_hi":0.6553902116402116,"top100":56,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.564781746031746,"hallu_rate_mean":0.5,"hallu_prevalence":0.55,"hallu_top100":87,"inter_rater_agreement":0.7523809523809524,"n_judges":3,"tokens_in_total":6530,"tokens_out_total":4177,"model_cost_usd":0.049933},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.3324514991181658,"score_ci_lo":0.235352733686067,"score_ci_hi":0.43516754850088185,"top100":33,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3324514991181658,"hallu_rate_mean":1.7111111111111112,"hallu_prevalence":0.8666666666666667,"hallu_top100":57,"inter_rater_agreement":0.6546560846560846,"n_judges":3,"tokens_in_total":5420,"tokens_out_total":3370,"model_cost_usd":0.040475},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.48493827160493824,"score_ci_lo":0.4106878306878307,"score_ci_hi":0.5618386243386244,"top100":48,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.48493827160493824,"hallu_rate_mean":1.2888888888888888,"hallu_prevalence":0.7666666666666667,"hallu_top100":67,"inter_rater_agreement":0.6157671957671957,"n_judges":3,"tokens_in_total":11545,"tokens_out_total":8050,"model_cost_usd":0.094931},{"model_id":"gpt-5","domain":"jura","score_mean":0.7444633408919122,"score_ci_lo":0.6849924414210128,"score_ci_hi":0.7936545729402872,"top100":74,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7444633408919122,"hallu_rate_mean":0.8476190476190476,"hallu_prevalence":0.7142857142857143,"hallu_top100":78,"inter_rater_agreement":0.7028684807256236,"n_judges":3,"tokens_in_total":11850,"tokens_out_total":59797,"model_cost_usd":0.956205},{"model_id":"gpt-5","domain":"medizin","score_mean":0.792526455026455,"score_ci_lo":0.7066468253968254,"score_ci_hi":0.8564153439153438,"top100":79,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.792526455026455,"hallu_rate_mean":1.05,"hallu_prevalence":0.9,"hallu_top100":73,"inter_rater_agreement":0.7704365079365079,"n_judges":3,"tokens_in_total":6251,"tokens_out_total":25865,"model_cost_usd":0.41923},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.6109435626102293,"score_ci_lo":0.505079365079365,"score_ci_hi":0.724973544973545,"top100":61,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.6109435626102293,"hallu_rate_mean":1.911111111111111,"hallu_prevalence":1,"hallu_top100":52,"inter_rater_agreement":0.6332539682539682,"n_judges":3,"tokens_in_total":5093,"tokens_out_total":39124,"model_cost_usd":0.612325},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7844268077601411,"score_ci_lo":0.7225308641975309,"score_ci_hi":0.8411860670194004,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7844268077601411,"hallu_rate_mean":1.5333333333333334,"hallu_prevalence":0.9333333333333333,"hallu_top100":61,"inter_rater_agreement":0.7159523809523809,"n_judges":3,"tokens_in_total":11129,"tokens_out_total":70561,"model_cost_usd":1.11406},{"model_id":"grok-4.3","domain":"jura","score_mean":0.48245275888133027,"score_ci_lo":0.41501889644746787,"score_ci_hi":0.5412698412698412,"top100":48,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.48245275888133027,"hallu_rate_mean":0.5428571428571428,"hallu_prevalence":0.4857142857142857,"hallu_top100":86,"inter_rater_agreement":0.6806802721088435,"n_judges":3,"tokens_in_total":17561,"tokens_out_total":4111,"model_cost_usd":0.032229},{"model_id":"grok-4.3","domain":"medizin","score_mean":0.5467592592592593,"score_ci_lo":0.47384259259259254,"score_ci_hi":0.6166666666666667,"top100":55,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.5467592592592593,"hallu_rate_mean":0.4166666666666667,"hallu_prevalence":0.5,"hallu_top100":89,"inter_rater_agreement":0.7301587301587301,"n_judges":3,"tokens_in_total":9055,"tokens_out_total":3723,"model_cost_usd":0.020626},{"model_id":"grok-4.3","domain":"steuerrecht","score_mean":0.3291710758377425,"score_ci_lo":0.23981481481481481,"score_ci_hi":0.41451499118165785,"top100":33,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.3291710758377425,"hallu_rate_mean":1.5333333333333334,"hallu_prevalence":0.8,"hallu_top100":61,"inter_rater_agreement":0.6142857142857142,"n_judges":3,"tokens_in_total":7535,"tokens_out_total":3507,"model_cost_usd":0.018186},{"model_id":"grok-4.3","domain":"wirtschaftsrecht","score_mean":0.4737345679012346,"score_ci_lo":0.4093783068783069,"score_ci_hi":0.5354144620811288,"top100":47,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.4737345679012346,"hallu_rate_mean":1.2888888888888888,"hallu_prevalence":0.8333333333333334,"hallu_top100":67,"inter_rater_agreement":0.5510185185185185,"n_judges":3,"tokens_in_total":16294,"tokens_out_total":8300,"model_cost_usd":0.041118},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.48307634164777025,"score_ci_lo":0.4071693121693121,"score_ci_hi":0.5589153439153439,"top100":48,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.48307634164777025,"hallu_rate_mean":1.5714285714285714,"hallu_prevalence":0.8,"hallu_top100":60,"inter_rater_agreement":0.6625056689342403,"n_judges":3,"tokens_in_total":13332,"tokens_out_total":10672,"model_cost_usd":0.090696},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6581349206349206,"score_ci_lo":0.5927579365079365,"score_ci_hi":0.7270833333333333,"top100":66,"n_cells":20,"closed_accuracy":null,"open_rubric_score":0.6581349206349206,"hallu_rate_mean":1.2166666666666666,"hallu_prevalence":0.95,"hallu_top100":69,"inter_rater_agreement":0.7142857142857143,"n_judges":3,"tokens_in_total":6669,"tokens_out_total":10857,"model_cost_usd":0.07848},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.25996472663139325,"score_ci_lo":0.172336860670194,"score_ci_hi":0.3524955908289241,"top100":26,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.25996472663139325,"hallu_rate_mean":3.311111111111111,"hallu_prevalence":1,"hallu_top100":17,"inter_rater_agreement":0.5866931216931217,"n_judges":3,"tokens_in_total":5572,"tokens_out_total":7088,"model_cost_usd":0.053672},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5404938271604939,"score_ci_lo":0.46402557319223986,"score_ci_hi":0.6086507936507937,"top100":54,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5404938271604939,"hallu_rate_mean":2.411111111111111,"hallu_prevalence":0.9666666666666667,"hallu_top100":39,"inter_rater_agreement":0.5254761904761904,"n_judges":3,"tokens_in_total":11870,"tokens_out_total":18900,"model_cost_usd":0.13714}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.793952380952381,"score_ci_lo":0.765276455026455,"score_ci_hi":0.8215608465608466,"top100":79,"n_cells":100,"hallu_rate_mean_across_domains":1.7066666666666666,"hallu_top100_across_domains":57},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.5212156084656084,"score_ci_lo":0.47760582010582014,"score_ci_hi":0.5633743386243386,"top100":52,"n_cells":100,"hallu_rate_mean_across_domains":0.9033333333333333,"hallu_top100_across_domains":77},{"model_id":"gpt-5","score_mean_across_domains":0.746037037037037,"score_ci_lo":0.7085410052910054,"score_ci_hi":0.781223544973545,"top100":75,"n_cells":100,"hallu_rate_mean_across_domains":1.2533333333333332,"hallu_top100_across_domains":68},{"model_id":"grok-4.3","score_mean_across_domains":0.46970634920634924,"score_ci_lo":0.4330515873015873,"score_ci_hi":0.5050119047619047,"top100":47,"n_cells":100,"hallu_rate_mean_across_domains":0.89,"hallu_top100_across_domains":77},{"model_id":"mistral-large-2","score_mean_across_domains":0.5018465608465609,"score_ci_lo":0.45565079365079364,"score_ci_hi":0.5465952380952381,"top100":50,"n_cells":100,"hallu_rate_mean_across_domains":2.0133333333333336,"hallu_top100_across_domains":49}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.8531,"oracle_top100":85,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8243,"best_single_model_top100":82},{"domain":"medizin","n_items":20,"oracle_mean":0.9083,"oracle_top100":91,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.8678,"best_single_model_top100":87},{"domain":"steuerrecht","n_items":15,"oracle_mean":0.7909,"oracle_top100":79,"best_single_model_id":"claude-opus-4-7","best_single_model_mean":0.7637,"best_single_model_top100":76},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8326,"oracle_top100":83,"best_single_model_id":"gpt-5","best_single_model_mean":0.7844,"best_single_model_top100":78}],"cost":{"model_calls_usd":13.196,"judge_calls_usd":63.6607,"total_usd":76.8567,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":1356495,"tokens_out_total":300841,"cost_usd":42.9105},{"judge_id":"gpt-5","tokens_in_total":702283,"tokens_out_total":917186,"cost_usd":17.269205},{"judge_id":"mistral-large-2","tokens_in_total":753437,"tokens_out_total":329027,"cost_usd":3.481036}],"extrapolation":{"scale_factor":12,"extrapolated_total_usd":922.28,"target_full_run_cells":1200,"observed_cells":100,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260528T202802-6ce096/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260528T202802-6ce096/progress.jsonl","cost_total_usd":76.8567}