{"run_id":"run-20260510T175034-5e294d","started_at":1778435434,"summary":{"run_id":"run-20260510T175034-5e294d","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778435434.0744092,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.8167162698412698,"score_ci_lo":0.7583498677248677,"score_ci_hi":0.8715277777777778,"top100":82,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.8167162698412698,"hallu_rate_mean":1.375,"hallu_prevalence":1,"hallu_top100":65,"inter_rater_agreement":0.7571924603174603,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":7498,"model_cost_usd":0.65688},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8209876543209876,"score_ci_lo":0.7685185185185185,"score_ci_hi":0.8672839506172839,"top100":82,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8209876543209876,"hallu_rate_mean":1.2222222222222223,"hallu_prevalence":1,"hallu_top100":69,"inter_rater_agreement":0.8302469135802469,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9236,"model_cost_usd":0.775275},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.6802083333333333,"score_ci_lo":0.6270833333333333,"score_ci_hi":0.7333333333333333,"top100":68,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6802083333333333,"hallu_rate_mean":2.4583333333333335,"hallu_prevalence":1,"hallu_top100":38,"inter_rater_agreement":0.8133928571428571,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10814,"model_cost_usd":0.897735},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7375,"score_ci_lo":0.6541666666666667,"score_ci_hi":0.8208333333333333,"top100":74,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7375,"hallu_rate_mean":2.7083333333333335,"hallu_prevalence":1,"hallu_top100":32,"inter_rater_agreement":0.775,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":14634,"model_cost_usd":1.22211},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.1650132275132275,"score_ci_lo":0.07407407407407407,"score_ci_hi":0.33994708994708994,"top100":17,"n_cells":4,"closed_accuracy":null,"open_rubric_score":0.1650132275132275,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.7827380952380952,"n_judges":3,"tokens_in_total":1866,"tokens_out_total":218,"model_cost_usd":0.004513},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.2851851851851852,"score_ci_lo":0.18518518518518517,"score_ci_hi":0.4,"top100":29,"n_cells":5,"closed_accuracy":null,"open_rubric_score":0.2851851851851852,"hallu_rate_mean":0.2,"hallu_prevalence":0.4,"hallu_top100":95,"inter_rater_agreement":0.8777777777777779,"n_judges":3,"tokens_in_total":1536,"tokens_out_total":577,"model_cost_usd":0.00769},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0,"score_ci_lo":0,"score_ci_hi":0,"top100":0,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":1,"n_judges":3,"tokens_in_total":456,"tokens_out_total":4,"model_cost_usd":0.00061},{"model_id":"gemini-2-5-pro","domain":"wirtschaftsrecht","score_mean":0.19999999999999998,"score_ci_lo":0.19999999999999998,"score_ci_hi":0.19999999999999998,"top100":20,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0.19999999999999998,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.8,"n_judges":3,"tokens_in_total":580,"tokens_out_total":54,"model_cost_usd":0.001265},{"model_id":"gpt-5","domain":"jura","score_mean":0.6421130952380952,"score_ci_lo":0.4919808201058201,"score_ci_hi":0.7847222222222222,"top100":64,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.6421130952380952,"hallu_rate_mean":0.9583333333333334,"hallu_prevalence":0.75,"hallu_top100":76,"inter_rater_agreement":0.6974206349206349,"n_judges":3,"tokens_in_total":3254,"tokens_out_total":14675,"model_cost_usd":0.236395},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8348765432098766,"score_ci_lo":0.7335390946502057,"score_ci_hi":0.9176954732510288,"top100":83,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8348765432098766,"hallu_rate_mean":0.8888888888888888,"hallu_prevalence":1,"hallu_top100":77,"inter_rater_agreement":0.7175925925925926,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":12962,"model_cost_usd":0.209405},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.4212797619047619,"score_ci_lo":0.2635416666666667,"score_ci_hi":0.5677083333333334,"top100":42,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4212797619047619,"hallu_rate_mean":3.0416666666666665,"hallu_prevalence":1,"hallu_top100":23,"inter_rater_agreement":0.6955357142857143,"n_judges":3,"tokens_in_total":3000,"tokens_out_total":21870,"model_cost_usd":0.34305},{"model_id":"gpt-5","domain":"wirtschaftsrecht","score_mean":0.7583333333333333,"score_ci_lo":0.6916666666666667,"score_ci_hi":0.8291666666666666,"top100":76,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7583333333333333,"hallu_rate_mean":1.5416666666666667,"hallu_prevalence":1,"hallu_top100":61,"inter_rater_agreement":0.7375,"n_judges":3,"tokens_in_total":4212,"tokens_out_total":20955,"model_cost_usd":0.335385},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.5278604497354498,"score_ci_lo":0.49016203703703703,"score_ci_hi":0.5620039682539683,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5278604497354498,"hallu_rate_mean":1.4166666666666667,"hallu_prevalence":1,"hallu_top100":64,"inter_rater_agreement":0.6314484126984127,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2858,"model_cost_usd":0.024544},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6815843621399177,"score_ci_lo":0.6147119341563786,"score_ci_hi":0.7453703703703703,"top100":68,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6815843621399177,"hallu_rate_mean":1.3333333333333333,"hallu_prevalence":1,"hallu_top100":66,"inter_rater_agreement":0.75,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5444,"model_cost_usd":0.03905},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.06636904761904762,"score_ci_lo":0.016369047619047616,"score_ci_hi":0.13333333333333333,"top100":7,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.06636904761904762,"hallu_rate_mean":4.5,"hallu_prevalence":1,"hallu_top100":0,"inter_rater_agreement":0.5620535714285714,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3616,"model_cost_usd":0.028214},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.525,"score_ci_lo":0.4208333333333333,"score_ci_hi":0.6375,"top100":52,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.525,"hallu_rate_mean":2.7083333333333335,"hallu_prevalence":1,"hallu_top100":32,"inter_rater_agreement":0.55,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6331,"model_cost_usd":0.046958}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7655844155844155,"score_ci_lo":0.7267396184062851,"score_ci_hi":0.8028058361391696,"top100":77,"n_cells":33,"hallu_rate_mean_across_domains":1.9191919191919193,"hallu_top100_across_domains":52},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.20781625781625782,"score_ci_lo":0.12667147667147668,"score_ci_hi":0.29523809523809524,"top100":21,"n_cells":11,"hallu_rate_mean_across_domains":0.09090909090909091,"hallu_top100_across_domains":97},{"model_id":"gpt-5","score_mean_across_domains":0.6693241943241943,"score_ci_lo":0.5888488055154721,"score_ci_hi":0.7487493987493987,"top100":67,"n_cells":33,"hallu_rate_mean_across_domains":1.5858585858585859,"hallu_top100_across_domains":60},{"model_id":"mistral-large-2","score_mean_across_domains":0.4572150072150072,"score_ci_lo":0.3732764149430816,"score_ci_hi":0.548733365400032,"top100":46,"n_cells":33,"hallu_rate_mean_across_domains":2.4545454545454546,"hallu_top100_across_domains":38}],"cost":{"model_calls_usd":4.8291,"judge_calls_usd":16.4013,"total_usd":21.2303,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":363862,"tokens_out_total":75691,"cost_usd":11.134755},{"judge_id":"gpt-5","tokens_in_total":189410,"tokens_out_total":229354,"cost_usd":4.38736},{"judge_id":"mistral-large-2","tokens_in_total":202497,"tokens_out_total":79024,"cost_usd":0.879138}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":772.01,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T175034-5e294d/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T175034-5e294d/progress.jsonl","cost_total_usd":21.2303}