{"run_id":"run-20260910T064154-bada0b","started_at":1789022514,"summary":{"run_id":"run-20260910T064154-bada0b","question_bank_version":"2026-09-09-v5","code_version":"0.2.13","official":true,"started_at":1789022514.8059044,"judges_active":["gpt-5","claude-opus-4-8","claude-opus-4-7","mistral-large-2"],"by_model_domain":[{"model_id":"claude-fable-5-1","domain":"jura","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.9074007936507936,"score_ci_lo":0.8799603174603174,"score_ci_hi":0.9336507936507936,"top100":91,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.9074007936507936,"hallu_rate_mean":1.6857142857142857,"hallu_prevalence":1,"hallu_per_1k_tokens":0.5991490053111005,"contradicted_rate_mean":0.02857142857142857,"extra_claims_rate_mean":1.6571428571428573,"extra_false_rate_mean":0.5642857142857143,"extra_unsupported_rate_mean":1.0928571428571427,"hallu_top100":57,"inter_rater_agreement":0.8689002267573696,"n_judges":4,"score_mean_ex_family":0.898344671201814,"self_judge_delta":0.009056122448979576,"n_cells_ex_family":35,"false_claim_types":{"az":16,"paragraph":125,"study":0,"number":2,"other":47},"tokens_in_total":22926,"tokens_out_total":98473},{"model_id":"claude-fable-5-1","domain":"medizin","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.8767625661375662,"score_ci_lo":0.8290740740740741,"score_ci_hi":0.9164417989417989,"top100":88,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8767625661375662,"hallu_rate_mean":1.5,"hallu_prevalence":1,"hallu_per_1k_tokens":0.7092422140965831,"contradicted_rate_mean":0.16666666666666666,"extra_claims_rate_mean":1.3333333333333333,"extra_false_rate_mean":0.30833333333333335,"extra_unsupported_rate_mean":1.025,"hallu_top100":62,"inter_rater_agreement":0.8400132275132275,"n_judges":4,"score_mean_ex_family":0.8566269841269841,"self_judge_delta":0.020135582010582076,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":51,"other":91},"tokens_in_total":17883,"tokens_out_total":63448},{"model_id":"claude-fable-5-1","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.8091674230787134,"score_ci_lo":0.740820648885165,"score_ci_hi":0.866442652329749,"top100":81,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.8091674230787134,"hallu_rate_mean":4,"hallu_prevalence":1,"hallu_per_1k_tokens":0.6659148273454703,"contradicted_rate_mean":0.3790322580645161,"extra_claims_rate_mean":3.620967741935484,"extra_false_rate_mean":1.7258064516129032,"extra_unsupported_rate_mean":1.8951612903225807,"hallu_top100":0,"inter_rater_agreement":0.8319508448540707,"n_judges":4,"score_mean_ex_family":0.7878956616859842,"self_judge_delta":0.021271761392729127,"n_cells_ex_family":31,"false_claim_types":{"az":11,"paragraph":223,"study":0,"number":44,"other":94},"tokens_in_total":20250,"tokens_out_total":186210},{"model_id":"claude-fable-5-1","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.7334093915343914,"score_ci_lo":0.6546064814814815,"score_ci_hi":0.8086805555555556,"top100":73,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7334093915343914,"hallu_rate_mean":3.7444444444444445,"hallu_prevalence":1,"hallu_per_1k_tokens":0.8136146460293432,"contradicted_rate_mean":0.5694444444444444,"extra_claims_rate_mean":3.175,"extra_false_rate_mean":1.5527777777777778,"extra_unsupported_rate_mean":1.6222222222222222,"hallu_top100":6,"inter_rater_agreement":0.7120899470899471,"n_judges":4,"score_mean_ex_family":0.679702380952381,"self_judge_delta":0.05370701058201044,"n_cells_ex_family":30,"false_claim_types":{"az":4,"paragraph":231,"study":0,"number":12,"other":72},"tokens_in_total":22171,"tokens_out_total":138067},{"model_id":"claude-opus-4-7","domain":"jura","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.7913775510204082,"score_ci_lo":0.7303174603174603,"score_ci_hi":0.8492261904761905,"top100":79,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7913775510204082,"hallu_rate_mean":1.3071428571428572,"hallu_prevalence":0.9142857142857143,"hallu_per_1k_tokens":1.4779996123279704,"contradicted_rate_mean":0.2,"extra_claims_rate_mean":1.1071428571428572,"extra_false_rate_mean":0.6285714285714286,"extra_unsupported_rate_mean":0.4785714285714286,"hallu_top100":67,"inter_rater_agreement":0.8209297052154194,"n_judges":4,"score_mean_ex_family":0.8017970521541951,"self_judge_delta":-0.010419501133786868,"n_cells_ex_family":35,"false_claim_types":{"az":2,"paragraph":90,"study":0,"number":5,"other":29},"tokens_in_total":23031,"tokens_out_total":30954},{"model_id":"claude-opus-4-7","domain":"medizin","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.8509160052910053,"score_ci_lo":0.803670634920635,"score_ci_hi":0.8937037037037037,"top100":85,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8509160052910053,"hallu_rate_mean":1.25,"hallu_prevalence":1,"hallu_per_1k_tokens":1.2481278082875686,"contradicted_rate_mean":0.10833333333333334,"extra_claims_rate_mean":1.1416666666666666,"extra_false_rate_mean":0.26666666666666666,"extra_unsupported_rate_mean":0.875,"hallu_top100":68,"inter_rater_agreement":0.8476455026455026,"n_judges":4,"score_mean_ex_family":0.822063492063492,"self_judge_delta":0.028852513227513255,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":2,"number":37,"other":85},"tokens_in_total":17973,"tokens_out_total":30045},{"model_id":"claude-opus-4-7","domain":"steuerrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.7321847507331378,"score_ci_lo":0.6933630661453242,"score_ci_hi":0.7685035842293906,"top100":73,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.7321847507331378,"hallu_rate_mean":3.0483870967741935,"hallu_prevalence":1,"hallu_per_1k_tokens":2.3084815321477428,"contradicted_rate_mean":0.3870967741935484,"extra_claims_rate_mean":2.661290322580645,"extra_false_rate_mean":1.4516129032258065,"extra_unsupported_rate_mean":1.2096774193548387,"hallu_top100":23,"inter_rater_agreement":0.7766117395149653,"n_judges":4,"score_mean_ex_family":0.7271866126704836,"self_judge_delta":0.004998138062654256,"n_cells_ex_family":31,"false_claim_types":{"az":11,"paragraph":143,"study":0,"number":40,"other":68},"tokens_in_total":20343,"tokens_out_total":40936},{"model_id":"claude-opus-4-7","domain":"wirtschaftsrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.7806084656084656,"score_ci_lo":0.7346230158730158,"score_ci_hi":0.8243088624338625,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7806084656084656,"hallu_rate_mean":2.9916666666666667,"hallu_prevalence":1,"hallu_per_1k_tokens":2.0591946770677985,"contradicted_rate_mean":0.24166666666666667,"extra_claims_rate_mean":2.75,"extra_false_rate_mean":1.525,"extra_unsupported_rate_mean":1.225,"hallu_top100":25,"inter_rater_agreement":0.7299338624338624,"n_judges":4,"score_mean_ex_family":0.7458068783068783,"self_judge_delta":0.03480158730158733,"n_cells_ex_family":30,"false_claim_types":{"az":10,"paragraph":185,"study":0,"number":25,"other":62},"tokens_in_total":22261,"tokens_out_total":43585},{"model_id":"claude-opus-4-8","domain":"jura","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.7978883219954648,"score_ci_lo":0.7592375283446712,"score_ci_hi":0.835249433106576,"top100":80,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7978883219954648,"hallu_rate_mean":0.7357142857142858,"hallu_prevalence":0.8,"hallu_per_1k_tokens":0.7992178528197648,"contradicted_rate_mean":0.04285714285714286,"extra_claims_rate_mean":0.6928571428571428,"extra_false_rate_mean":0.2785714285714286,"extra_unsupported_rate_mean":0.4142857142857143,"hallu_top100":81,"inter_rater_agreement":0.7739455782312925,"n_judges":4,"score_mean_ex_family":0.809767573696145,"self_judge_delta":-0.011879251700680271,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":51,"study":0,"number":7,"other":29},"tokens_in_total":22856,"tokens_out_total":32219},{"model_id":"claude-opus-4-8","domain":"medizin","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.8063624338624339,"score_ci_lo":0.7477314814814815,"score_ci_hi":0.861111111111111,"top100":81,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8063624338624339,"hallu_rate_mean":0.85,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":0.8728094194961665,"contradicted_rate_mean":0.15,"extra_claims_rate_mean":0.7,"extra_false_rate_mean":0.1,"extra_unsupported_rate_mean":0.6,"hallu_top100":78,"inter_rater_agreement":0.7923677248677249,"n_judges":4,"score_mean_ex_family":0.7773941798941799,"self_judge_delta":0.028968253968253976,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":16,"other":64},"tokens_in_total":17823,"tokens_out_total":29216},{"model_id":"claude-opus-4-8","domain":"steuerrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.7106107154494252,"score_ci_lo":0.6589550458502071,"score_ci_hi":0.7590126495368431,"top100":71,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.7106107154494252,"hallu_rate_mean":1.9516129032258065,"hallu_prevalence":1,"hallu_per_1k_tokens":1.3765329571568339,"contradicted_rate_mean":0.27419354838709675,"extra_claims_rate_mean":1.6774193548387097,"extra_false_rate_mean":0.9112903225806451,"extra_unsupported_rate_mean":0.7661290322580645,"hallu_top100":51,"inter_rater_agreement":0.7760496671786995,"n_judges":4,"score_mean_ex_family":0.6949041102266909,"self_judge_delta":0.01570660522273437,"n_cells_ex_family":31,"false_claim_types":{"az":1,"paragraph":77,"study":0,"number":34,"other":67},"tokens_in_total":20188,"tokens_out_total":43951},{"model_id":"claude-opus-4-8","domain":"wirtschaftsrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.6732936507936508,"score_ci_lo":0.6181944444444445,"score_ci_hi":0.7353869047619047,"top100":67,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6732936507936508,"hallu_rate_mean":2.35,"hallu_prevalence":1,"hallu_per_1k_tokens":1.5578733371635658,"contradicted_rate_mean":0.475,"extra_claims_rate_mean":1.875,"extra_false_rate_mean":1.0083333333333333,"extra_unsupported_rate_mean":0.8666666666666667,"hallu_top100":41,"inter_rater_agreement":0.6863756613756614,"n_judges":4,"score_mean_ex_family":0.624378306878307,"self_judge_delta":0.0489153439153438,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":136,"study":0,"number":10,"other":57},"tokens_in_total":22111,"tokens_out_total":45254},{"model_id":"claude-opus-5","domain":"jura","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.9106632653061225,"score_ci_lo":0.8756632653061225,"score_ci_hi":0.9394302721088434,"top100":91,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.9106632653061225,"hallu_rate_mean":2.05,"hallu_prevalence":1,"hallu_per_1k_tokens":0.6017578878507808,"contradicted_rate_mean":0.12142857142857143,"extra_claims_rate_mean":1.9285714285714286,"extra_false_rate_mean":0.55,"extra_unsupported_rate_mean":1.3785714285714286,"hallu_top100":48,"inter_rater_agreement":0.8553741496598639,"n_judges":4,"score_mean_ex_family":0.8980839002267573,"self_judge_delta":0.01257936507936519,"n_cells_ex_family":35,"false_claim_types":{"az":10,"paragraph":162,"study":0,"number":8,"other":54},"tokens_in_total":22856,"tokens_out_total":119234},{"model_id":"claude-opus-5","domain":"medizin","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.9200231481481481,"score_ci_lo":0.8680092592592593,"score_ci_hi":0.9584027777777777,"top100":92,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.9200231481481481,"hallu_rate_mean":2.05,"hallu_prevalence":1,"hallu_per_1k_tokens":0.6523676171079429,"contradicted_rate_mean":0.19166666666666668,"extra_claims_rate_mean":1.8583333333333334,"extra_false_rate_mean":0.325,"extra_unsupported_rate_mean":1.5333333333333334,"hallu_top100":48,"inter_rater_agreement":0.9001851851851852,"n_judges":4,"score_mean_ex_family":0.9076388888888889,"self_judge_delta":0.012384259259259234,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":66,"other":139},"tokens_in_total":17823,"tokens_out_total":94272},{"model_id":"claude-opus-5","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.8899467020434763,"score_ci_lo":0.8527684680910488,"score_ci_hi":0.9222964087883443,"top100":89,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.8899467020434763,"hallu_rate_mean":4.25,"hallu_prevalence":1,"hallu_per_1k_tokens":0.5896648644777829,"contradicted_rate_mean":0.3225806451612903,"extra_claims_rate_mean":3.9274193548387095,"extra_false_rate_mean":1.8387096774193548,"extra_unsupported_rate_mean":2.088709677419355,"hallu_top100":0,"inter_rater_agreement":0.8625168738071963,"n_judges":4,"score_mean_ex_family":0.8579335986594051,"self_judge_delta":0.03201310338407115,"n_cells_ex_family":31,"false_claim_types":{"az":32,"paragraph":238,"study":0,"number":72,"other":76},"tokens_in_total":20188,"tokens_out_total":223432},{"model_id":"claude-opus-5","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.7988756613756614,"score_ci_lo":0.7377116402116403,"score_ci_hi":0.8581746031746033,"top100":80,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7988756613756614,"hallu_rate_mean":3.7333333333333334,"hallu_prevalence":1,"hallu_per_1k_tokens":0.47434311948364366,"contradicted_rate_mean":0.5083333333333333,"extra_claims_rate_mean":3.225,"extra_false_rate_mean":1.375,"extra_unsupported_rate_mean":1.85,"hallu_top100":6,"inter_rater_agreement":0.7937698412698412,"n_judges":4,"score_mean_ex_family":0.7484060846560846,"self_judge_delta":0.05046957671957675,"n_cells_ex_family":30,"false_claim_types":{"az":8,"paragraph":209,"study":0,"number":23,"other":102},"tokens_in_total":22111,"tokens_out_total":236116},{"model_id":"deepseek-v4-pro","domain":"jura","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.6103769841269842,"score_ci_lo":0.5350396825396825,"score_ci_hi":0.6817261904761904,"top100":61,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6103769841269842,"hallu_rate_mean":0.7357142857142858,"hallu_prevalence":0.6285714285714286,"hallu_per_1k_tokens":0.11004320531284321,"contradicted_rate_mean":0.34285714285714286,"extra_claims_rate_mean":0.39285714285714285,"extra_false_rate_mean":0.18571428571428572,"extra_unsupported_rate_mean":0.20714285714285716,"hallu_top100":81,"inter_rater_agreement":0.7232993197278912,"n_judges":4,"score_mean_ex_family":0.6103769841269842,"self_judge_delta":0,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":21,"study":0,"number":3,"other":19},"tokens_in_total":16575,"tokens_out_total":233999},{"model_id":"deepseek-v4-pro","domain":"medizin","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.6796329365079364,"score_ci_lo":0.5998611111111111,"score_ci_hi":0.7420403439153439,"top100":68,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6796329365079364,"hallu_rate_mean":0.5333333333333333,"hallu_prevalence":0.8,"hallu_per_1k_tokens":0.243605359317905,"contradicted_rate_mean":0.19166666666666668,"extra_claims_rate_mean":0.3416666666666667,"extra_false_rate_mean":0.125,"extra_unsupported_rate_mean":0.21666666666666667,"hallu_top100":86,"inter_rater_agreement":0.7192195767195767,"n_judges":4,"score_mean_ex_family":0.6796329365079364,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":8,"other":32},"tokens_in_total":13028,"tokens_out_total":65680},{"model_id":"deepseek-v4-pro","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.48025851836335703,"score_ci_lo":0.41330441511893123,"score_ci_hi":0.5442402131918261,"top100":48,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.48025851836335703,"hallu_rate_mean":2.2983870967741935,"hallu_prevalence":0.9354838709677419,"hallu_per_1k_tokens":0.25702999585144565,"contradicted_rate_mean":0.8467741935483871,"extra_claims_rate_mean":1.4516129032258065,"extra_false_rate_mean":0.967741935483871,"extra_unsupported_rate_mean":0.4838709677419355,"hallu_top100":42,"inter_rater_agreement":0.648220686123912,"n_judges":4,"score_mean_ex_family":0.48025851836335703,"self_judge_delta":0,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":43,"study":0,"number":52,"other":44},"tokens_in_total":14442,"tokens_out_total":277205},{"model_id":"deepseek-v4-pro","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.5910119047619048,"score_ci_lo":0.5153075396825397,"score_ci_hi":0.6657837301587303,"top100":59,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5910119047619048,"hallu_rate_mean":1.9416666666666667,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":0.2517231693351483,"contradicted_rate_mean":0.5,"extra_claims_rate_mean":1.4416666666666667,"extra_false_rate_mean":0.9,"extra_unsupported_rate_mean":0.5416666666666666,"hallu_top100":51,"inter_rater_agreement":0.6271957671957672,"n_judges":4,"score_mean_ex_family":0.5910119047619048,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":69,"study":0,"number":15,"other":59},"tokens_in_total":15332,"tokens_out_total":231405},{"model_id":"gemini-3-1-pro","domain":"jura","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.5862018140589569,"score_ci_lo":0.5202579365079365,"score_ci_hi":0.6447732426303855,"top100":59,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.5862018140589569,"hallu_rate_mean":0.5785714285714286,"hallu_prevalence":0.5428571428571428,"hallu_per_1k_tokens":3.118263012011087,"contradicted_rate_mean":0.24285714285714285,"extra_claims_rate_mean":0.3357142857142857,"extra_false_rate_mean":0.15714285714285714,"extra_unsupported_rate_mean":0.17857142857142858,"hallu_top100":85,"inter_rater_agreement":0.6827097505668934,"n_judges":4,"score_mean_ex_family":0.5862018140589569,"self_judge_delta":0,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":15,"study":0,"number":6,"other":17},"tokens_in_total":12953,"tokens_out_total":6494},{"model_id":"gemini-3-1-pro","domain":"medizin","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.5962863756613757,"score_ci_lo":0.5426322751322752,"score_ci_hi":0.6498974867724867,"top100":60,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5962863756613757,"hallu_rate_mean":0.4,"hallu_prevalence":0.5666666666666667,"hallu_per_1k_tokens":1.9014419267944858,"contradicted_rate_mean":0.11666666666666667,"extra_claims_rate_mean":0.2833333333333333,"extra_false_rate_mean":0.09166666666666666,"extra_unsupported_rate_mean":0.19166666666666668,"hallu_top100":90,"inter_rater_agreement":0.7326587301587302,"n_judges":4,"score_mean_ex_family":0.5962863756613757,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":3,"other":28},"tokens_in_total":10224,"tokens_out_total":6311},{"model_id":"gemini-3-1-pro","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.48600870455709166,"score_ci_lo":0.43226766513056836,"score_ci_hi":0.5432219662058372,"top100":49,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.48600870455709166,"hallu_rate_mean":1.0403225806451613,"hallu_prevalence":0.7419354838709677,"hallu_per_1k_tokens":4.217340133385641,"contradicted_rate_mean":0.4112903225806452,"extra_claims_rate_mean":0.6290322580645161,"extra_false_rate_mean":0.3790322580645161,"extra_unsupported_rate_mean":0.25,"hallu_top100":73,"inter_rater_agreement":0.674204021784667,"n_judges":4,"score_mean_ex_family":0.48600870455709166,"self_judge_delta":0,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":14,"study":0,"number":10,"other":36},"tokens_in_total":11090,"tokens_out_total":7647},{"model_id":"gemini-3-1-pro","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean":0.48924272486772485,"score_ci_lo":0.4187962962962963,"score_ci_hi":0.5568022486772487,"top100":49,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.48924272486772485,"hallu_rate_mean":1.5416666666666667,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":5.070160052620039,"contradicted_rate_mean":0.6611111111111111,"extra_claims_rate_mean":0.8805555555555556,"extra_false_rate_mean":0.5166666666666667,"extra_unsupported_rate_mean":0.3638888888888889,"hallu_top100":61,"inter_rater_agreement":0.5750925925925926,"n_judges":4,"score_mean_ex_family":0.48924272486772485,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":27,"study":0,"number":14,"other":45},"tokens_in_total":11544,"tokens_out_total":9122},{"model_id":"glm-5.3","domain":"jura","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.7036892361111111,"score_ci_lo":0.5969184027777777,"score_ci_hi":0.7900607638888889,"top100":70,"n_cells":32,"closed_accuracy":null,"open_rubric_score":0.7036892361111111,"hallu_rate_mean":2.7421875,"hallu_prevalence":1,"hallu_per_1k_tokens":0.32208808512668796,"contradicted_rate_mean":0.7109375,"extra_claims_rate_mean":2.03125,"extra_false_rate_mean":1.0859375,"extra_unsupported_rate_mean":0.9453125,"hallu_top100":31,"inter_rater_agreement":0.7931423611111111,"n_judges":4,"score_mean_ex_family":0.7036892361111111,"self_judge_delta":0,"n_cells_ex_family":32,"false_claim_types":{"az":4,"paragraph":148,"study":0,"number":9,"other":57},"tokens_in_total":12069,"tokens_out_total":272441},{"model_id":"glm-5.3","domain":"medizin","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.8677281746031745,"score_ci_lo":0.819212962962963,"score_ci_hi":0.907691798941799,"top100":87,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8677281746031745,"hallu_rate_mean":1.5916666666666666,"hallu_prevalence":1,"hallu_per_1k_tokens":0.34690941849989826,"contradicted_rate_mean":0.18333333333333332,"extra_claims_rate_mean":1.4083333333333334,"extra_false_rate_mean":0.44166666666666665,"extra_unsupported_rate_mean":0.9666666666666667,"hallu_top100":60,"inter_rater_agreement":0.7932142857142856,"n_judges":4,"score_mean_ex_family":0.8677281746031745,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":51,"other":101},"tokens_in_total":10855,"tokens_out_total":137644},{"model_id":"glm-5.3","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.5850308641975308,"score_ci_lo":0.4219907407407407,"score_ci_hi":0.7095679012345679,"top100":59,"n_cells":9,"closed_accuracy":null,"open_rubric_score":0.5850308641975308,"hallu_rate_mean":5.083333333333333,"hallu_prevalence":1,"hallu_per_1k_tokens":0.5374828181722059,"contradicted_rate_mean":1.2777777777777777,"extra_claims_rate_mean":3.8055555555555554,"extra_false_rate_mean":2.9722222222222223,"extra_unsupported_rate_mean":0.8333333333333334,"hallu_top100":0,"inter_rater_agreement":0.7266273849607183,"n_judges":4,"score_mean_ex_family":0.5850308641975308,"self_judge_delta":0,"n_cells_ex_family":9,"false_claim_types":{"az":13,"paragraph":43,"study":0,"number":19,"other":34},"tokens_in_total":3054,"tokens_out_total":85119},{"model_id":"glm-5.3","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.6755140692640693,"score_ci_lo":0.5471861471861472,"score_ci_hi":0.78125,"top100":68,"n_cells":11,"closed_accuracy":null,"open_rubric_score":0.6755140692640693,"hallu_rate_mean":4.204545454545454,"hallu_prevalence":1,"hallu_per_1k_tokens":0.35317475468672443,"contradicted_rate_mean":0.6363636363636364,"extra_claims_rate_mean":3.5681818181818183,"extra_false_rate_mean":2.3636363636363638,"extra_unsupported_rate_mean":1.2045454545454546,"hallu_top100":0,"inter_rater_agreement":0.728030303030303,"n_judges":4,"score_mean_ex_family":0.6755140692640693,"self_judge_delta":0,"n_cells_ex_family":11,"false_claim_types":{"az":1,"paragraph":96,"study":0,"number":7,"other":18},"tokens_in_total":4133,"tokens_out_total":130955},{"model_id":"gpt-5","domain":"jura","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7691213151927437,"score_ci_lo":0.7040419501133787,"score_ci_hi":0.832687074829932,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7691213151927437,"hallu_rate_mean":1.2,"hallu_prevalence":0.8571428571428571,"hallu_per_1k_tokens":0.2814070351758794,"contradicted_rate_mean":0.22142857142857142,"extra_claims_rate_mean":0.9785714285714285,"extra_false_rate_mean":0.42142857142857143,"extra_unsupported_rate_mean":0.5571428571428572,"hallu_top100":70,"inter_rater_agreement":0.7687755102040816,"n_judges":4,"score_mean_ex_family":0.7741836734693878,"self_judge_delta":-0.00506235827664403,"n_cells_ex_family":35,"false_claim_types":{"az":2,"paragraph":65,"study":0,"number":6,"other":40},"tokens_in_total":11892,"tokens_out_total":149250},{"model_id":"gpt-5","domain":"medizin","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.8404431216931217,"score_ci_lo":0.7750033068783069,"score_ci_hi":0.8958630952380953,"top100":84,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8404431216931217,"hallu_rate_mean":1.3,"hallu_prevalence":1,"hallu_per_1k_tokens":0.407255411797876,"contradicted_rate_mean":0.25833333333333336,"extra_claims_rate_mean":1.0416666666666667,"extra_false_rate_mean":0.19166666666666668,"extra_unsupported_rate_mean":0.85,"hallu_top100":67,"inter_rater_agreement":0.8324206349206349,"n_judges":4,"score_mean_ex_family":0.8704629629629629,"self_judge_delta":-0.03001984126984114,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":56,"other":60},"tokens_in_total":9789,"tokens_out_total":95763},{"model_id":"gpt-5","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.6502202322766839,"score_ci_lo":0.5660580458967556,"score_ci_hi":0.733786016850533,"top100":65,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.6502202322766839,"hallu_rate_mean":2.725806451612903,"hallu_prevalence":1,"hallu_per_1k_tokens":0.4467259481692166,"contradicted_rate_mean":0.7661290322580645,"extra_claims_rate_mean":1.9596774193548387,"extra_false_rate_mean":1.1290322580645162,"extra_unsupported_rate_mean":0.8306451612903226,"hallu_top100":31,"inter_rater_agreement":0.7209642508029604,"n_judges":4,"score_mean_ex_family":0.677467066983196,"self_judge_delta":-0.027246834706512124,"n_cells_ex_family":31,"false_claim_types":{"az":1,"paragraph":83,"study":0,"number":63,"other":50},"tokens_in_total":10385,"tokens_out_total":189154},{"model_id":"gpt-5","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7965806878306878,"score_ci_lo":0.721468253968254,"score_ci_hi":0.8598048941798943,"top100":80,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7965806878306878,"hallu_rate_mean":1.825,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":0.33019123952886686,"contradicted_rate_mean":0.3416666666666667,"extra_claims_rate_mean":1.4833333333333334,"extra_false_rate_mean":0.5916666666666667,"extra_unsupported_rate_mean":0.8916666666666667,"hallu_top100":54,"inter_rater_agreement":0.7485449735449735,"n_judges":4,"score_mean_ex_family":0.8354938271604938,"self_judge_delta":-0.038913139329806,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":82,"study":0,"number":15,"other":60},"tokens_in_total":11128,"tokens_out_total":165813},{"model_id":"gpt-5.6-sol","domain":"jura","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7746995464852607,"score_ci_lo":0.71906179138322,"score_ci_hi":0.8182539682539682,"top100":77,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7746995464852607,"hallu_rate_mean":0.9214285714285714,"hallu_prevalence":0.8285714285714286,"hallu_per_1k_tokens":0.5453069782384471,"contradicted_rate_mean":0.12857142857142856,"extra_claims_rate_mean":0.7928571428571428,"extra_false_rate_mean":0.3,"extra_unsupported_rate_mean":0.4928571428571429,"hallu_top100":76,"inter_rater_agreement":0.7580725623582766,"n_judges":4,"score_mean_ex_family":0.7944217687074829,"self_judge_delta":-0.019722222222222197,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":68,"study":0,"number":4,"other":17},"tokens_in_total":11892,"tokens_out_total":59141},{"model_id":"gpt-5.6-sol","domain":"medizin","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.8326951058201059,"score_ci_lo":0.7862632275132275,"score_ci_hi":0.8765013227513226,"top100":83,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8326951058201059,"hallu_rate_mean":0.75,"hallu_prevalence":0.9,"hallu_per_1k_tokens":0.7755678880424666,"contradicted_rate_mean":0.175,"extra_claims_rate_mean":0.575,"extra_false_rate_mean":0.125,"extra_unsupported_rate_mean":0.45,"hallu_top100":81,"inter_rater_agreement":0.7788095238095238,"n_judges":4,"score_mean_ex_family":0.8774911816578483,"self_judge_delta":-0.04479607583774248,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":16,"other":46},"tokens_in_total":9789,"tokens_out_total":29011},{"model_id":"gpt-5.6-sol","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7316765000232742,"score_ci_lo":0.6751504096262161,"score_ci_hi":0.780130801098543,"top100":73,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.7316765000232742,"hallu_rate_mean":1.7741935483870968,"hallu_prevalence":1,"hallu_per_1k_tokens":0.524173949507753,"contradicted_rate_mean":0.3064516129032258,"extra_claims_rate_mean":1.467741935483871,"extra_false_rate_mean":0.7258064516129032,"extra_unsupported_rate_mean":0.7419354838709677,"hallu_top100":55,"inter_rater_agreement":0.780657962109575,"n_judges":4,"score_mean_ex_family":0.7575800245155084,"self_judge_delta":-0.02590352449223421,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":64,"study":0,"number":40,"other":49},"tokens_in_total":10385,"tokens_out_total":104927},{"model_id":"gpt-5.6-sol","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7791997354497354,"score_ci_lo":0.7217195767195768,"score_ci_hi":0.8358862433862434,"top100":78,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7791997354497354,"hallu_rate_mean":1.4666666666666666,"hallu_prevalence":1,"hallu_per_1k_tokens":0.557145389621901,"contradicted_rate_mean":0.3416666666666667,"extra_claims_rate_mean":1.125,"extra_false_rate_mean":0.5833333333333334,"extra_unsupported_rate_mean":0.5416666666666666,"hallu_top100":63,"inter_rater_agreement":0.7302380952380952,"n_judges":4,"score_mean_ex_family":0.8321957671957672,"self_judge_delta":-0.052996031746031735,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":55,"study":0,"number":10,"other":57},"tokens_in_total":11128,"tokens_out_total":78974},{"model_id":"gpt-6-astra","domain":"jura","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7343820861678004,"score_ci_lo":0.6677380952380952,"score_ci_hi":0.7941468253968255,"top100":73,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.7343820861678004,"hallu_rate_mean":1.2857142857142858,"hallu_prevalence":0.9142857142857143,"hallu_per_1k_tokens":1.2257572455872738,"contradicted_rate_mean":0.2571428571428571,"extra_claims_rate_mean":1.0285714285714285,"extra_false_rate_mean":0.5071428571428571,"extra_unsupported_rate_mean":0.5214285714285715,"hallu_top100":67,"inter_rater_agreement":0.6884353741496598,"n_judges":4,"score_mean_ex_family":0.7718820861678004,"self_judge_delta":-0.03749999999999998,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":82,"study":0,"number":13,"other":30},"tokens_in_total":11892,"tokens_out_total":36712},{"model_id":"gpt-6-astra","domain":"medizin","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.81676917989418,"score_ci_lo":0.7808597883597883,"score_ci_hi":0.8532539682539683,"top100":82,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.81676917989418,"hallu_rate_mean":0.9666666666666667,"hallu_prevalence":0.9,"hallu_per_1k_tokens":1.1398474962660168,"contradicted_rate_mean":0.26666666666666666,"extra_claims_rate_mean":0.7,"extra_false_rate_mean":0.175,"extra_unsupported_rate_mean":0.525,"hallu_top100":75,"inter_rater_agreement":0.7779761904761905,"n_judges":4,"score_mean_ex_family":0.8605776014109348,"self_judge_delta":-0.04380842151675479,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":21,"other":57},"tokens_in_total":9789,"tokens_out_total":25442},{"model_id":"gpt-6-astra","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.736117220593027,"score_ci_lo":0.6750692407950473,"score_ci_hi":0.7930840664711633,"top100":74,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.736117220593027,"hallu_rate_mean":1.8790322580645162,"hallu_prevalence":1,"hallu_per_1k_tokens":0.9758426589827782,"contradicted_rate_mean":0.31451612903225806,"extra_claims_rate_mean":1.564516129032258,"extra_false_rate_mean":0.6612903225806451,"extra_unsupported_rate_mean":0.9032258064516129,"hallu_top100":53,"inter_rater_agreement":0.7220150816925011,"n_judges":4,"score_mean_ex_family":0.7723591521978619,"self_judge_delta":-0.03624193160483491,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":43,"study":0,"number":50,"other":76},"tokens_in_total":10385,"tokens_out_total":59692},{"model_id":"gpt-6-astra","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean":0.7544775132275132,"score_ci_lo":0.6872718253968254,"score_ci_hi":0.824785052910053,"top100":75,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7544775132275132,"hallu_rate_mean":1.575,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":0.9308143887159687,"contradicted_rate_mean":0.3333333333333333,"extra_claims_rate_mean":1.2416666666666667,"extra_false_rate_mean":0.6333333333333333,"extra_unsupported_rate_mean":0.6083333333333333,"hallu_top100":60,"inter_rater_agreement":0.7002777777777778,"n_judges":4,"score_mean_ex_family":0.8068121693121693,"self_judge_delta":-0.05233465608465604,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":63,"study":0,"number":17,"other":59},"tokens_in_total":11128,"tokens_out_total":50762},{"model_id":"grok-4.5","domain":"jura","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.652562358276644,"score_ci_lo":0.5891496598639455,"score_ci_hi":0.7033985260770975,"top100":65,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.652562358276644,"hallu_rate_mean":0.5642857142857143,"hallu_prevalence":0.5142857142857142,"hallu_per_1k_tokens":2.8364210828665803,"contradicted_rate_mean":0.20714285714285716,"extra_claims_rate_mean":0.35714285714285715,"extra_false_rate_mean":0.2,"extra_unsupported_rate_mean":0.15714285714285714,"hallu_top100":85,"inter_rater_agreement":0.7015192743764173,"n_judges":4,"score_mean_ex_family":0.652562358276644,"self_judge_delta":0,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":22,"study":0,"number":3,"other":14},"tokens_in_total":28950,"tokens_out_total":6963},{"model_id":"grok-4.5","domain":"medizin","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.7016798941798942,"score_ci_lo":0.6465674603174603,"score_ci_hi":0.7590906084656085,"top100":70,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.7016798941798942,"hallu_rate_mean":0.45,"hallu_prevalence":0.7666666666666667,"hallu_per_1k_tokens":1.5919811320754718,"contradicted_rate_mean":0.13333333333333333,"extra_claims_rate_mean":0.31666666666666665,"extra_false_rate_mean":0.058333333333333334,"extra_unsupported_rate_mean":0.25833333333333336,"hallu_top100":88,"inter_rater_agreement":0.7005423280423281,"n_judges":4,"score_mean_ex_family":0.7016798941798942,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":7,"other":28},"tokens_in_total":24284,"tokens_out_total":8480},{"model_id":"grok-4.5","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.5796818996415771,"score_ci_lo":0.5270833333333333,"score_ci_hi":0.6291890681003585,"top100":58,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.5796818996415771,"hallu_rate_mean":1.25,"hallu_prevalence":0.967741935483871,"hallu_per_1k_tokens":3.613727501632006,"contradicted_rate_mean":0.3548387096774194,"extra_claims_rate_mean":0.8951612903225806,"extra_false_rate_mean":0.4596774193548387,"extra_unsupported_rate_mean":0.43548387096774194,"hallu_top100":68,"inter_rater_agreement":0.7045547642321837,"n_judges":4,"score_mean_ex_family":0.5796818996415771,"self_judge_delta":0,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":29,"study":0,"number":9,"other":56},"tokens_in_total":25335,"tokens_out_total":10723},{"model_id":"grok-4.5","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.6658498677248678,"score_ci_lo":0.6113293650793651,"score_ci_hi":0.7097420634920635,"top100":67,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6658498677248678,"hallu_rate_mean":1,"hallu_prevalence":0.9,"hallu_per_1k_tokens":2.279981760145919,"contradicted_rate_mean":0.225,"extra_claims_rate_mean":0.775,"extra_false_rate_mean":0.30833333333333335,"extra_unsupported_rate_mean":0.4666666666666667,"hallu_top100":75,"inter_rater_agreement":0.6003042328042327,"n_judges":4,"score_mean_ex_family":0.6658498677248678,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":31,"study":0,"number":7,"other":44},"tokens_in_total":25514,"tokens_out_total":13158},{"model_id":"grok-4.6","domain":"jura","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.6437018140589569,"score_ci_lo":0.5582341269841269,"score_ci_hi":0.7121626984126984,"top100":64,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.6437018140589569,"hallu_rate_mean":0.65,"hallu_prevalence":0.5142857142857142,"hallu_per_1k_tokens":3.360413589364845,"contradicted_rate_mean":0.2857142857142857,"extra_claims_rate_mean":0.36428571428571427,"extra_false_rate_mean":0.22857142857142856,"extra_unsupported_rate_mean":0.1357142857142857,"hallu_top100":83,"inter_rater_agreement":0.7156462585034014,"n_judges":4,"score_mean_ex_family":0.6437018140589569,"self_judge_delta":0,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":18,"study":0,"number":3,"other":20},"tokens_in_total":33920,"tokens_out_total":6770},{"model_id":"grok-4.6","domain":"medizin","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.6444246031746032,"score_ci_lo":0.5780357142857143,"score_ci_hi":0.7027579365079365,"top100":64,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6444246031746032,"hallu_rate_mean":0.5583333333333333,"hallu_prevalence":0.7333333333333333,"hallu_per_1k_tokens":2.4156331122007497,"contradicted_rate_mean":0.15833333333333333,"extra_claims_rate_mean":0.4,"extra_false_rate_mean":0.075,"extra_unsupported_rate_mean":0.325,"hallu_top100":86,"inter_rater_agreement":0.7429365079365079,"n_judges":4,"score_mean_ex_family":0.6444246031746032,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":0,"number":6,"other":38},"tokens_in_total":28544,"tokens_out_total":6934},{"model_id":"grok-4.6","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.5252894730717311,"score_ci_lo":0.46132697947214074,"score_ci_hi":0.5838002723083369,"top100":53,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.5252894730717311,"hallu_rate_mean":1.9112903225806452,"hallu_prevalence":0.967741935483871,"hallu_per_1k_tokens":4.723750298971538,"contradicted_rate_mean":0.6451612903225806,"extra_claims_rate_mean":1.2661290322580645,"extra_false_rate_mean":0.7419354838709677,"extra_unsupported_rate_mean":0.5241935483870968,"hallu_top100":52,"inter_rater_agreement":0.6860587441232603,"n_judges":4,"score_mean_ex_family":0.5252894730717311,"self_judge_delta":0,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":60,"study":0,"number":16,"other":55},"tokens_in_total":29737,"tokens_out_total":12543},{"model_id":"grok-4.6","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean":0.6898148148148149,"score_ci_lo":0.6242890211640212,"score_ci_hi":0.7511607142857143,"top100":69,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6898148148148149,"hallu_rate_mean":1.4166666666666667,"hallu_prevalence":0.9666666666666667,"hallu_per_1k_tokens":2.7708958143173814,"contradicted_rate_mean":0.375,"extra_claims_rate_mean":1.0416666666666667,"extra_false_rate_mean":0.5333333333333333,"extra_unsupported_rate_mean":0.5083333333333333,"hallu_top100":64,"inter_rater_agreement":0.659589947089947,"n_judges":4,"score_mean_ex_family":0.6898148148148149,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":58,"study":0,"number":4,"other":50},"tokens_in_total":29774,"tokens_out_total":15338},{"model_id":"kimi-k3","domain":"jura","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.8114408263305322,"score_ci_lo":0.7615254435107376,"score_ci_hi":0.8600898692810458,"top100":81,"n_cells":34,"closed_accuracy":null,"open_rubric_score":0.8114408263305322,"hallu_rate_mean":1.9705882352941178,"hallu_prevalence":0.8823529411764706,"hallu_per_1k_tokens":0.3951869765247139,"contradicted_rate_mean":0.2377450980392157,"extra_claims_rate_mean":1.7328431372549018,"extra_false_rate_mean":0.9191176470588235,"extra_unsupported_rate_mean":0.8137254901960784,"hallu_top100":50,"inter_rater_agreement":0.7906979458450047,"n_judges":4,"score_mean_ex_family":0.8114408263305322,"self_judge_delta":0,"n_cells_ex_family":34,"false_claim_types":{"az":3,"paragraph":150,"study":0,"number":6,"other":38},"tokens_in_total":17543,"tokens_out_total":169540},{"model_id":"kimi-k3","domain":"medizin","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.8658796296296296,"score_ci_lo":0.8127314814814814,"score_ci_hi":0.9106018518518518,"top100":87,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.8658796296296296,"hallu_rate_mean":1.5583333333333333,"hallu_prevalence":1,"hallu_per_1k_tokens":0.4993644452514981,"contradicted_rate_mean":0.225,"extra_claims_rate_mean":1.3333333333333333,"extra_false_rate_mean":0.225,"extra_unsupported_rate_mean":1.1083333333333334,"hallu_top100":61,"inter_rater_agreement":0.836574074074074,"n_judges":4,"score_mean_ex_family":0.8658796296296296,"self_judge_delta":0,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":50,"other":91},"tokens_in_total":14166,"tokens_out_total":93619},{"model_id":"kimi-k3","domain":"steuerrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.5776292688792689,"score_ci_lo":0.4326785714285714,"score_ci_hi":0.7121464646464646,"top100":58,"n_cells":15,"closed_accuracy":null,"open_rubric_score":0.5776292688792689,"hallu_rate_mean":4.216666666666667,"hallu_prevalence":1,"hallu_per_1k_tokens":0.4896041366711563,"contradicted_rate_mean":1.1666666666666667,"extra_claims_rate_mean":3.05,"extra_false_rate_mean":1.9166666666666667,"extra_unsupported_rate_mean":1.1333333333333333,"hallu_top100":0,"inter_rater_agreement":0.7193650793650794,"n_judges":4,"score_mean_ex_family":0.5776292688792689,"self_judge_delta":0,"n_cells_ex_family":15,"false_claim_types":{"az":6,"paragraph":71,"study":0,"number":35,"other":36},"tokens_in_total":7520,"tokens_out_total":129186},{"model_id":"kimi-k3","domain":"wirtschaftsrecht","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean":0.738558201058201,"score_ci_lo":0.652980324074074,"score_ci_hi":0.8154513888888889,"top100":74,"n_cells":24,"closed_accuracy":null,"open_rubric_score":0.738558201058201,"hallu_rate_mean":3.59375,"hallu_prevalence":1,"hallu_per_1k_tokens":0.5029301145806001,"contradicted_rate_mean":0.5208333333333334,"extra_claims_rate_mean":3.0729166666666665,"extra_false_rate_mean":1.875,"extra_unsupported_rate_mean":1.1979166666666667,"hallu_top100":10,"inter_rater_agreement":0.7278935185185186,"n_judges":4,"score_mean_ex_family":0.738558201058201,"self_judge_delta":0,"n_cells_ex_family":24,"false_claim_types":{"az":0,"paragraph":176,"study":0,"number":17,"other":66},"tokens_in_total":12966,"tokens_out_total":171495},{"model_id":"mistral-large-2","domain":"jura","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.4273044217687075,"score_ci_lo":0.3487131519274376,"score_ci_hi":0.5079790249433107,"top100":43,"n_cells":35,"closed_accuracy":null,"open_rubric_score":0.4273044217687075,"hallu_rate_mean":2.05,"hallu_prevalence":0.8857142857142857,"hallu_per_1k_tokens":7.147838214783821,"contradicted_rate_mean":0.9714285714285714,"extra_claims_rate_mean":1.0785714285714285,"extra_false_rate_mean":0.8357142857142857,"extra_unsupported_rate_mean":0.24285714285714285,"hallu_top100":48,"inter_rater_agreement":0.6412018140589569,"n_judges":4,"score_mean_ex_family":0.3830801209372638,"self_judge_delta":0.044224300831443686,"n_cells_ex_family":35,"false_claim_types":{"az":0,"paragraph":58,"study":0,"number":3,"other":47},"tokens_in_total":13389,"tokens_out_total":10038},{"model_id":"mistral-large-2","domain":"medizin","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.6523908730158731,"score_ci_lo":0.6025859788359789,"score_ci_hi":0.7019378306878307,"top100":65,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.6523908730158731,"hallu_rate_mean":1.4583333333333333,"hallu_prevalence":1,"hallu_per_1k_tokens":2.8549986948577395,"contradicted_rate_mean":0.25833333333333336,"extra_claims_rate_mean":1.2,"extra_false_rate_mean":0.3,"extra_unsupported_rate_mean":0.9,"hallu_top100":63,"inter_rater_agreement":0.693968253968254,"n_judges":4,"score_mean_ex_family":0.6189550264550264,"self_judge_delta":0.03343584656084664,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":0,"study":1,"number":27,"other":96},"tokens_in_total":10461,"tokens_out_total":15324},{"model_id":"mistral-large-2","domain":"steuerrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.27157956291020807,"score_ci_lo":0.20617814085556022,"score_ci_hi":0.3329338895871154,"top100":27,"n_cells":31,"closed_accuracy":null,"open_rubric_score":0.27157956291020807,"hallu_rate_mean":3.2661290322580645,"hallu_prevalence":1,"hallu_per_1k_tokens":7.222341108495613,"contradicted_rate_mean":1.403225806451613,"extra_claims_rate_mean":1.8629032258064515,"extra_false_rate_mean":1.3951612903225807,"extra_unsupported_rate_mean":0.46774193548387094,"hallu_top100":18,"inter_rater_agreement":0.5291416468835825,"n_judges":4,"score_mean_ex_family":0.22399804496578687,"self_judge_delta":0.0475815179444212,"n_cells_ex_family":31,"false_claim_types":{"az":0,"paragraph":60,"study":0,"number":38,"other":86},"tokens_in_total":11379,"tokens_out_total":14019},{"model_id":"mistral-large-2","domain":"wirtschaftsrecht","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean":0.5887962962962963,"score_ci_lo":0.5191269841269841,"score_ci_hi":0.6550099206349207,"top100":59,"n_cells":30,"closed_accuracy":null,"open_rubric_score":0.5887962962962963,"hallu_rate_mean":2.441666666666667,"hallu_prevalence":1,"hallu_per_1k_tokens":3.9263507718696395,"contradicted_rate_mean":0.35833333333333334,"extra_claims_rate_mean":2.0833333333333335,"extra_false_rate_mean":1.3666666666666667,"extra_unsupported_rate_mean":0.7166666666666667,"hallu_top100":38,"inter_rater_agreement":0.5571031746031746,"n_judges":4,"score_mean_ex_family":0.5389638447971782,"self_judge_delta":0.04983245149911808,"n_cells_ex_family":30,"false_claim_types":{"az":0,"paragraph":102,"study":0,"number":25,"other":77},"tokens_in_total":11869,"tokens_out_total":18656}],"by_model_overall":[{"model_id":"claude-fable-5-1","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean_across_domains":0.8345109255823542,"score_ci_lo":0.8036186696900982,"score_ci_hi":0.8626193911908198,"top100":83,"n_cells":126,"hallu_rate_mean_across_domains":2.701058201058201,"hallu_top100_across_domains":32,"hallu_per_1k_tokens":0.6999891676504908,"contradicted_rate_mean":0.27645502645502645,"extra_claims_rate_mean":2.4246031746031744,"extra_false_rate_mean":1.0244708994708995,"extra_unsupported_rate_mean":1.4001322751322751,"score_mean_ex_family":0.8091802377516664,"self_judge_delta":0.025330687830687815,"n_cells_ex_family":126,"false_claim_types":{"az":31,"paragraph":579,"study":1,"number":109,"other":304}},{"model_id":"claude-opus-4-7","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean_across_domains":0.7884259975033785,"score_ci_lo":0.7623575608694657,"score_ci_hi":0.8135047327011613,"top100":79,"n_cells":126,"hallu_rate_mean_across_domains":2.123015873015873,"hallu_top100_across_domains":46,"hallu_per_1k_tokens":1.838235294117647,"contradicted_rate_mean":0.23412698412698413,"extra_claims_rate_mean":1.8888888888888888,"extra_false_rate_mean":0.9583333333333334,"extra_unsupported_rate_mean":0.9305555555555556,"score_mean_ex_family":0.7749348645181979,"self_judge_delta":0.013491132985180565,"n_cells_ex_family":126,"false_claim_types":{"az":23,"paragraph":418,"study":2,"number":107,"other":244}},{"model_id":"claude-opus-4-8","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean_across_domains":0.7487675078448888,"score_ci_lo":0.7210482804232805,"score_ci_hi":0.7747367381295953,"top100":75,"n_cells":126,"hallu_rate_mean_across_domains":1.4464285714285714,"hallu_top100_across_domains":63,"hallu_per_1k_tokens":1.2098380244291025,"contradicted_rate_mean":0.22817460317460317,"extra_claims_rate_mean":1.2182539682539681,"extra_false_rate_mean":0.5654761904761905,"extra_unsupported_rate_mean":0.6527777777777778,"score_mean_ex_family":0.729659262694977,"self_judge_delta":0.019108245149911718,"n_cells_ex_family":126,"false_claim_types":{"az":1,"paragraph":264,"study":1,"number":67,"other":217}},{"model_id":"claude-opus-5","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean_across_domains":0.8811787804347327,"score_ci_lo":0.8575373204837491,"score_ci_hi":0.9029813440527726,"top100":88,"n_cells":126,"hallu_rate_mean_across_domains":2.992063492063492,"hallu_top100_across_domains":25,"hallu_per_1k_tokens":0.5601333622562231,"contradicted_rate_mean":0.27976190476190477,"extra_claims_rate_mean":2.7123015873015874,"extra_false_rate_mean":1.0099206349206349,"extra_unsupported_rate_mean":1.7023809523809523,"score_mean_ex_family":0.8548430735930735,"self_judge_delta":0.0263357068416592,"n_cells_ex_family":126,"false_claim_types":{"az":50,"paragraph":609,"study":1,"number":169,"other":371}},{"model_id":"deepseek-v4-pro","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean_across_domains":0.5902424900936806,"score_ci_lo":0.5524864575459814,"score_ci_hi":0.6268118400558876,"top100":59,"n_cells":126,"hallu_rate_mean_across_domains":1.3591269841269842,"hallu_top100_across_domains":66,"hallu_per_1k_tokens":0.21186729004106206,"contradicted_rate_mean":0.46825396825396826,"extra_claims_rate_mean":0.8908730158730159,"extra_false_rate_mean":0.5337301587301587,"extra_unsupported_rate_mean":0.35714285714285715,"score_mean_ex_family":0.5902424900936806,"self_judge_delta":0,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":133,"study":0,"number":78,"other":154}},{"model_id":"gemini-3-1-pro","reasoning_mode":"on","reasoning_level":"adaptive","reasoning_vendor_default":true,"score_mean_across_domains":0.5408667170571932,"score_ci_lo":0.5121488410178886,"score_ci_hi":0.5729292642983119,"top100":54,"n_cells":126,"hallu_rate_mean_across_domains":0.878968253968254,"hallu_top100_across_domains":78,"hallu_per_1k_tokens":3.7448434435652938,"contradicted_rate_mean":0.35383597883597884,"extra_claims_rate_mean":0.5251322751322751,"extra_false_rate_mean":0.28174603174603174,"extra_unsupported_rate_mean":0.2433862433862434,"score_mean_ex_family":0.5408667170571932,"self_judge_delta":0,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":56,"study":0,"number":33,"other":126}},{"model_id":"glm-5.3","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean_across_domains":0.746900406504065,"score_ci_lo":0.6975754936120789,"score_ci_hi":0.7941298877274487,"top100":75,"n_cells":82,"hallu_rate_mean_across_domains":2.774390243902439,"hallu_top100_across_domains":30,"hallu_per_1k_tokens":0.3633262478060684,"contradicted_rate_mean":0.5701219512195121,"extra_claims_rate_mean":2.2042682926829267,"extra_false_rate_mean":1.228658536585366,"extra_unsupported_rate_mean":0.975609756097561,"score_mean_ex_family":0.746900406504065,"self_judge_delta":0,"n_cells_ex_family":82,"false_claim_types":{"az":18,"paragraph":287,"study":1,"number":86,"other":210}},{"model_id":"gpt-5","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean_across_domains":0.7633872025241073,"score_ci_lo":0.7245212899974806,"score_ci_hi":0.8004344751368561,"top100":76,"n_cells":126,"hallu_rate_mean_across_domains":1.748015873015873,"hallu_top100_across_domains":56,"hallu_per_1k_tokens":0.36709556985232844,"contradicted_rate_mean":0.39285714285714285,"extra_claims_rate_mean":1.3551587301587302,"extra_false_rate_mean":0.5813492063492064,"extra_unsupported_rate_mean":0.7738095238095238,"score_mean_ex_family":0.7879096139016774,"self_judge_delta":-0.024522411377570053,"n_cells_ex_family":126,"false_claim_types":{"az":3,"paragraph":230,"study":0,"number":140,"other":210}},{"model_id":"gpt-5.6-sol","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean_across_domains":0.77899445131588,"score_ci_lo":0.7514158735884926,"score_ci_hi":0.8045219771410247,"top100":78,"n_cells":126,"hallu_rate_mean_across_domains":1.2202380952380953,"hallu_top100_across_domains":69,"hallu_per_1k_tokens":0.5651472323407571,"contradicted_rate_mean":0.23412698412698413,"extra_claims_rate_mean":0.9861111111111112,"extra_false_rate_mean":0.4305555555555556,"extra_unsupported_rate_mean":0.5555555555555556,"score_mean_ex_family":0.8141297708758026,"self_judge_delta":-0.03513531955992266,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":187,"study":1,"number":70,"other":169}},{"model_id":"gpt-6-astra","reasoning_mode":"on","reasoning_level":"medium","reasoning_vendor_default":true,"score_mean_across_domains":0.7592095845072035,"score_ci_lo":0.7308137354268306,"score_ci_hi":0.7875133134061706,"top100":76,"n_cells":126,"hallu_rate_mean_across_domains":1.4246031746031746,"hallu_top100_across_domains":64,"hallu_per_1k_tokens":1.0399286243974786,"contradicted_rate_mean":0.2916666666666667,"extra_claims_rate_mean":1.132936507936508,"extra_false_rate_mean":0.49603174603174605,"extra_unsupported_rate_mean":0.6369047619047619,"score_mean_ex_family":0.8014341258388877,"self_judge_delta":-0.042224541331684184,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":188,"study":0,"number":101,"other":222}},{"model_id":"grok-4.5","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean_across_domains":0.6494897959183674,"score_ci_lo":0.6216939090450995,"score_ci_hi":0.6787863756613757,"top100":65,"n_cells":126,"hallu_rate_mean_across_domains":0.8095238095238095,"hallu_top100_across_domains":79,"hallu_per_1k_tokens":2.593835825450107,"contradicted_rate_mean":0.23015873015873015,"extra_claims_rate_mean":0.5793650793650794,"extra_false_rate_mean":0.25595238095238093,"extra_unsupported_rate_mean":0.32341269841269843,"score_mean_ex_family":0.6494897959183674,"self_judge_delta":0,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":82,"study":0,"number":26,"other":142}},{"model_id":"grok-4.6","reasoning_mode":"on","reasoning_level":"high","reasoning_vendor_default":true,"score_mean_across_domains":0.6257199975949976,"score_ci_lo":0.5894202798964704,"score_ci_hi":0.6615236119700405,"top100":63,"n_cells":126,"hallu_rate_mean_across_domains":1.121031746031746,"hallu_top100_across_domains":71,"hallu_per_1k_tokens":3.3966574485992544,"contradicted_rate_mean":0.36507936507936506,"extra_claims_rate_mean":0.7559523809523809,"extra_false_rate_mean":0.39087301587301587,"extra_unsupported_rate_mean":0.36507936507936506,"score_mean_ex_family":0.6257199975949976,"self_judge_delta":0,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":136,"study":0,"number":29,"other":163}},{"model_id":"kimi-k3","reasoning_mode":"on","reasoning_level":"standard","reasoning_vendor_default":true,"score_mean_across_domains":0.7762642023564353,"score_ci_lo":0.7354826874851147,"score_ci_hi":0.8141816920943133,"top100":78,"n_cells":103,"hallu_rate_mean_across_domains":2.5558252427184467,"hallu_top100_across_domains":36,"hallu_per_1k_tokens":0.4668877695800227,"contradicted_rate_mean":0.4352750809061489,"extra_claims_rate_mean":2.1205501618122975,"extra_false_rate_mean":1.0849514563106797,"extra_unsupported_rate_mean":1.0355987055016183,"score_mean_ex_family":0.7762642023564353,"self_judge_delta":0,"n_cells_ex_family":103,"false_claim_types":{"az":9,"paragraph":397,"study":1,"number":108,"other":231}},{"model_id":"mistral-large-2","reasoning_mode":"off","reasoning_level":"off","reasoning_vendor_default":true,"score_mean_across_domains":0.4810336213610023,"score_ci_lo":0.43992676195057145,"score_ci_hi":0.5229719244600196,"top100":48,"n_cells":126,"hallu_rate_mean_across_domains":2.3015873015873014,"hallu_top100_across_domains":42,"hallu_per_1k_tokens":4.996812378310388,"contradicted_rate_mean":0.7619047619047619,"extra_claims_rate_mean":1.5396825396825398,"extra_false_rate_mean":0.9722222222222222,"extra_unsupported_rate_mean":0.5674603174603174,"score_mean_ex_family":0.43721674416118855,"self_judge_delta":0.04381687719981375,"n_cells_ex_family":126,"false_claim_types":{"az":0,"paragraph":220,"study":1,"number":93,"other":306}}],"oracle_by_domain":[{"domain":"jura","n_items":35,"oracle_mean":0.9536,"oracle_top100":95,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.9107,"best_single_model_top100":91},{"domain":"medizin","n_items":30,"oracle_mean":0.9604,"oracle_top100":96,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.92,"best_single_model_top100":92},{"domain":"steuerrecht","n_items":31,"oracle_mean":0.9175,"oracle_top100":92,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.8899,"best_single_model_top100":89},{"domain":"wirtschaftsrecht","n_items":30,"oracle_mean":0.8905,"oracle_top100":89,"best_single_model_id":"claude-opus-5","best_single_model_mean":0.7989,"best_single_model_top100":80}],"cost":{"total_usd":283.5037}},"raw_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260910T064154-bada0b/raw.jsonl","progress_jsonl_url":"https://licenses.ai-roundtable.de/r2/runs/run-20260910T064154-bada0b/progress.jsonl","cost_total_usd":283.5037}