{"run_id":"run-20260510T164305-2c7b16","started_at":1778431385,"summary":{"run_id":"run-20260510T164305-2c7b16","question_bank_version":"2026-05-08-pilot","code_version":"0.1.0","started_at":1778431385.2935522,"judges_active":["gpt-5","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-opus-4-7","domain":"jura","score_mean":0.7953042328042328,"score_ci_lo":0.7196593915343915,"score_ci_hi":0.8761574074074073,"top100":80,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7953042328042328,"hallu_rate_mean":1.0833333333333333,"hallu_prevalence":0.875,"hallu_top100":72,"inter_rater_agreement":0.7976190476190477,"n_judges":3,"tokens_in_total":6302,"tokens_out_total":7059,"model_cost_usd":0.623955},{"model_id":"claude-opus-4-7","domain":"medizin","score_mean":0.8590534979423868,"score_ci_lo":0.8060699588477367,"score_ci_hi":0.9166666666666666,"top100":86,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8590534979423868,"hallu_rate_mean":1.1111111111111112,"hallu_prevalence":1,"hallu_top100":72,"inter_rater_agreement":0.7901234567901234,"n_judges":3,"tokens_in_total":5505,"tokens_out_total":9099,"model_cost_usd":0.765},{"model_id":"claude-opus-4-7","domain":"steuerrecht","score_mean":0.7139880952380953,"score_ci_lo":0.6358630952380953,"score_ci_hi":0.7973214285714285,"top100":71,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7139880952380953,"hallu_rate_mean":2.5416666666666665,"hallu_prevalence":1,"hallu_top100":36,"inter_rater_agreement":0.8674107142857143,"n_judges":3,"tokens_in_total":5779,"tokens_out_total":10781,"model_cost_usd":0.89526},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","score_mean":0.7374999999999999,"score_ci_lo":0.6541666666666667,"score_ci_hi":0.825,"top100":74,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.7374999999999999,"hallu_rate_mean":2.7083333333333335,"hallu_prevalence":1,"hallu_top100":32,"inter_rater_agreement":0.7375,"n_judges":3,"tokens_in_total":8304,"tokens_out_total":14231,"model_cost_usd":1.191885},{"model_id":"gemini-2-5-pro","domain":"jura","score_mean":0.07407407407407407,"score_ci_lo":0.07407407407407407,"score_ci_hi":0.07407407407407407,"top100":7,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0.07407407407407407,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.8888888888888888,"n_judges":3,"tokens_in_total":479,"tokens_out_total":13,"model_cost_usd":0.000729},{"model_id":"gemini-2-5-pro","domain":"medizin","score_mean":0.18827160493827164,"score_ci_lo":0.125,"score_ci_hi":0.2916666666666667,"top100":19,"n_cells":3,"closed_accuracy":null,"open_rubric_score":0.18827160493827164,"hallu_rate_mean":0.1111111111111111,"hallu_prevalence":0.3333333333333333,"hallu_top100":97,"inter_rater_agreement":0.9212962962962963,"n_judges":3,"tokens_in_total":1233,"tokens_out_total":316,"model_cost_usd":0.004701},{"model_id":"gemini-2-5-pro","domain":"steuerrecht","score_mean":0.125,"score_ci_lo":0.125,"score_ci_hi":0.125,"top100":12,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0.125,"hallu_rate_mean":0.3333333333333333,"hallu_prevalence":1,"hallu_top100":91,"inter_rater_agreement":1,"n_judges":3,"tokens_in_total":449,"tokens_out_total":85,"model_cost_usd":0.001411},{"model_id":"gpt-5","domain":"jura","score_mean":0.6479276895943563,"score_ci_lo":0.548941798941799,"score_ci_hi":0.7669753086419752,"top100":65,"n_cells":6,"closed_accuracy":null,"open_rubric_score":0.6479276895943563,"hallu_rate_mean":0.5,"hallu_prevalence":0.6666666666666666,"hallu_top100":87,"inter_rater_agreement":0.5849867724867724,"n_judges":3,"tokens_in_total":2506,"tokens_out_total":10290,"model_cost_usd":0.16688},{"model_id":"gpt-5","domain":"medizin","score_mean":0.8225308641975309,"score_ci_lo":0.757201646090535,"score_ci_hi":0.8811728395061729,"top100":82,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.8225308641975309,"hallu_rate_mean":1,"hallu_prevalence":0.7777777777777778,"hallu_top100":75,"inter_rater_agreement":0.6774691358024691,"n_judges":3,"tokens_in_total":2995,"tokens_out_total":12869,"model_cost_usd":0.20801},{"model_id":"gpt-5","domain":"steuerrecht","score_mean":0.9047619047619048,"score_ci_lo":0.9047619047619048,"score_ci_hi":0.9047619047619048,"top100":90,"n_cells":1,"closed_accuracy":null,"open_rubric_score":0.9047619047619048,"hallu_rate_mean":0,"hallu_prevalence":0,"hallu_top100":100,"inter_rater_agreement":0.7142857142857143,"n_judges":3,"tokens_in_total":363,"tokens_out_total":1947,"model_cost_usd":0.03102},{"model_id":"mistral-large-2","domain":"jura","score_mean":0.4870205026455026,"score_ci_lo":0.4131944444444444,"score_ci_hi":0.5700231481481481,"top100":49,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.4870205026455026,"hallu_rate_mean":1.1666666666666667,"hallu_prevalence":0.75,"hallu_top100":70,"inter_rater_agreement":0.564484126984127,"n_judges":3,"tokens_in_total":3698,"tokens_out_total":2653,"model_cost_usd":0.023314},{"model_id":"mistral-large-2","domain":"medizin","score_mean":0.6589506172839505,"score_ci_lo":0.5540123456790123,"score_ci_hi":0.7561728395061729,"top100":66,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.6589506172839505,"hallu_rate_mean":1.7037037037037037,"hallu_prevalence":1,"hallu_top100":57,"inter_rater_agreement":0.6697530864197531,"n_judges":3,"tokens_in_total":3193,"tokens_out_total":5677,"model_cost_usd":0.040448},{"model_id":"mistral-large-2","domain":"steuerrecht","score_mean":0.11741071428571428,"score_ci_lo":0.08407738095238095,"score_ci_hi":0.14657738095238093,"top100":12,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.11741071428571428,"hallu_rate_mean":3.833333333333333,"hallu_prevalence":1,"hallu_top100":4,"inter_rater_agreement":0.5691964285714286,"n_judges":3,"tokens_in_total":3259,"tokens_out_total":3669,"model_cost_usd":0.028532},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","score_mean":0.5291666666666667,"score_ci_lo":0.4083333333333333,"score_ci_hi":0.6208333333333333,"top100":53,"n_cells":8,"closed_accuracy":null,"open_rubric_score":0.5291666666666667,"hallu_rate_mean":3.1666666666666665,"hallu_prevalence":1,"hallu_top100":20,"inter_rater_agreement":0.5875,"n_judges":3,"tokens_in_total":4486,"tokens_out_total":6775,"model_cost_usd":0.049622}],"by_model_overall":[{"model_id":"claude-opus-4-7","score_mean_across_domains":0.7789642456309123,"score_ci_lo":0.73670835337502,"score_ci_hi":0.8217051467051467,"top100":78,"n_cells":33,"hallu_rate_mean_across_domains":1.8383838383838382,"hallu_top100_across_domains":54},{"model_id":"gemini-2-5-pro","score_mean_across_domains":0.15277777777777776,"score_ci_lo":0.09907407407407406,"score_ci_hi":0.225,"top100":15,"n_cells":5,"hallu_rate_mean_across_domains":0.13333333333333333,"hallu_top100_across_domains":96},{"model_id":"gpt-5","score_mean_across_domains":0.7621941137566137,"score_ci_lo":0.6887814153439153,"score_ci_hi":0.8308531746031746,"top100":76,"n_cells":16,"hallu_rate_mean_across_domains":0.75,"hallu_top100_across_domains":81},{"model_id":"mistral-large-2","score_mean_across_domains":0.4545254128587462,"score_ci_lo":0.3758698092031425,"score_ci_hi":0.5363596280262948,"top100":45,"n_cells":33,"hallu_rate_mean_across_domains":2.4444444444444446,"hallu_top100_across_domains":38}],"cost":{"model_calls_usd":4.0308,"judge_calls_usd":12.6181,"total_usd":16.6489,"by_judge":[{"judge_id":"claude-opus-4-7","tokens_in_total":282046,"tokens_out_total":57420,"cost_usd":8.53719},{"judge_id":"gpt-5","tokens_in_total":147167,"tokens_out_total":177511,"cost_usd":3.3985},{"judge_id":"mistral-large-2","tokens_in_total":157088,"tokens_out_total":61375,"cost_usd":0.682426}],"extrapolation":{"scale_factor":36.36,"extrapolated_total_usd":605.41,"target_full_run_cells":1200,"observed_cells":33,"note":"linear extrapolation; assumes uniform prompt sizes + judge mix"},"missing_pricing_for":[]}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T164305-2c7b16/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260510T164305-2c7b16/progress.jsonl","cost_total_usd":16.6489}