{"run_id":"run-20260510T121001-e3a863","started_at":1778415001,"summary":{"run_id":"run-20260510T121001-e3a863","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778415001.8540955,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7771990740740741,"score_ci_lo":0.7037037037037037,"score_ci_hi":0.8547453703703703,"top100":78,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7771990740740741,"hallu_rate_mean":1.0833333333333333,"hallu_prevalence":0.875,"hallu_top100":72,"inter_rater_agreement":0.7889384920634921,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":6741,"model_cost_usd":0.600105},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8621399176954733,"score_ci_lo":0.7777777777777778,"score_ci_hi":0.9279835390946501,"top100":86,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8621399176954733,"hallu_rate_mean":1.3703703703703705,"hallu_prevalence":1,"hallu_top100":65,"inter_rater_agreement":0.8950617283950617,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":8606,"model_cost_usd":0.728025},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6636904761904763,"score_ci_lo":0.5928571428571429,"score_ci_hi":0.7325892857142857,"top100":66,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6636904761904763,"hallu_rate_mean":2.9166666666666665,"hallu_prevalence":1,"hallu_top100":27,"inter_rater_agreement":0.7696428571428572,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10787,"model_cost_usd":0.89571},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.775,"score_ci_lo":0.675,"score_ci_hi":0.875,"top100":78,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.775,"hallu_rate_mean":2.7916666666666665,"hallu_prevalence":1,"hallu_top100":30,"inter_rater_agreement":0.7,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":15280,"model_cost_usd":1.27056},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.3774250440917107,"score_ci_lo":0.1111111111111111,"score_ci_hi":0.5925925925925926,"top100":38,"n_cells":3,"closed_accuracy":null,"open_rubric_score":0.3774250440917107,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.7936507936507936,"n_judges":3,"tokens_in_total":1386,"tokens_out_total":380,"model_cost_usd":0.005533},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.4282407407407407,"score_ci_lo":0.27314814814814814,"score_ci_hi":0.5833333333333333,"top100":43,"n_cells":4,"closed_accuracy":null,"open_rubric_score":0.4282407407407407,"hallu_rate_mean":0.16666666666666666,"hallu_prevalence":0.5,"hallu_top100":95,"inter_rater_agreement":0.7847222222222222,"n_judges":3,"tokens_in_total":1224,"tokens_out_total":519,"model_cost_usd":0.00672},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0,"score_ci_lo":0,"score_ci_hi":0,"top100":0,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":1,"n_judges":3,"tokens_in_total":456,"tokens_out_total":4,"model_cost_usd":0.00061},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.06666666666666667,"score_ci_lo":0.06666666666666667,"score_ci_hi":0.06666666666666667,"top100":7,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0.06666666666666667,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.9,"n_judges":3,"tokens_in_total":698,"tokens_out_total":11,"model_cost_usd":0.000983},{"model_id":"gpt-5","domain":"jura","score_mean":0.7004913076341648,"score_ci_lo":0.6256613756613757,"score_ci_hi":0.7753212396069539,"top100":70,"n_cells":7,"closed_accuracy":null,"open_rubric_score":0.7004913076341648,"hallu_rate_mean":0.619047619047619,"hallu_prevalence":0.8571428571428571,"hallu_top100":84,"inter_rater_agreement":0.6091269841269842,"n_judges":3,"tokens_in_total":2915,"tokens_out_total":10418,"model_cost_usd":0.170845},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8431069958847737,"score_ci_lo":0.7834362139917695,"score_ci_hi":0.8986625514403292,"top100":84,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8431069958847737,"hallu_rate_mean":1.1481481481481481,"hallu_prevalence":0.8888888888888888,"hallu_top100":71,"inter_rater_agreement":0.787037037037037,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":11977,"model_cost_usd":0.19463},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.5329034391534392,"score_ci_lo":0.48817791005291,"score_ci_hi":0.5798611111111112,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5329034391534392,"hallu_rate_mean":1.25,"hallu_prevalence":0.875,"hallu_top100":68,"inter_rater_agreement":0.5840773809523809,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2510,"model_cost_usd":0.022456},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.683641975308642,"score_ci_lo":0.5776748971193416,"score_ci_hi":0.7659465020576133,"top100":68,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.683641975308642,"hallu_rate_mean":1.1851851851851851,"hallu_prevalence":0.8888888888888888,"hallu_top100":70,"inter_rater_agreement":0.7114197530864197,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5343,"model_cost_usd":0.038444},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.08854166666666666,"score_ci_lo":0.03229166666666666,"score_ci_hi":0.15312499999999998,"top100":9,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.08854166666666666,"hallu_rate_mean":4.166666666666667,"hallu_prevalence":1,"hallu_top100":0,"inter_rater_agreement":0.6361607142857143,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3952,"model_cost_usd":0.03023},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5,"score_ci_lo":0.3375,"score_ci_hi":0.6458333333333334,"top100":50,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5,"hallu_rate_mean":2.875,"hallu_prevalence":1,"hallu_top100":28,"inter_rater_agreement":0.5625,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":5704,"model_cost_usd":0.043196}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7723144139810806,"score_ci_lo":0.7213323713323714,"score_ci_hi":0.8222382555715889,"top100":77,"n_cells":33,"hallu_rate_mean_across_domains":2.0202020202020203,"hallu_top100_across_domains":49},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.3235449735449735,"score_ci_lo":0.19074074074074074,"score_ci_hi":0.4731040564373898,"top100":32,"n_cells":9,"hallu_rate_mean_across_domains":0.07407407407407407,"hallu_top100_across_domains":98},{"model_id":"gpt-5","score_mean_across_domains":0.7807126322751322,"score_ci_lo":0.7213955026455026,"score_ci_hi":0.8362681878306878,"top100":78,"n_cells":16,"hallu_rate_mean_across_domains":0.9166666666666666,"hallu_top100_across_domains":77},{"model_id":"mistral-large-2","score_mean_across_domains":0.458313291646625,"score_ci_lo":0.36402918069584733,"score_ci_hi":0.5450497033830367,"top100":46,"n_cells":33,"hallu_rate_mean_across_domains":2.3333333333333335,"hallu_top100_across_domains":41}],"cost":{"model_calls_usd":4.008,"judge_calls_usd":13.1237,"total_usd":17.1317,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":290051,"tokens_out_total":60512,"cost_usd":8.889165},{"judge_id":"gpt-5","tokens_in_total":151425,"tokens_out_total":184122,"cost_usd":3.518955},{"judge_id":"mistral-large-2","tokens_in_total":161712,"tokens_out_total":65358,"cost_usd":0.715572}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":622.97,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T121001-e3a863/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T121001-e3a863/progress.jsonl","cost_total_usd":17.1317}