{"id":"https://openalex.org/W7206170481","doi":"https://doi.org/10.48550/arxiv.2609.00482","title":"Are Near-Tied LLM Rankings Robust to Family-DIF-Guided Benchmark Recomposition?","display_name":"Are Near-Tied LLM Rankings Robust to Family-DIF-Guided Benchmark Recomposition?","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7206170481","doi":"https://doi.org/10.48550/arxiv.2609.00482"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.00482","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.00482","pdf_url":"https://arxiv.org/pdf/2609.00482","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.00482","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136599062","display_name":"Qiaoyuan Zheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Zheng, Qiaoyuan","raw_affiliation_strings":["ETH Zurich Zurich , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich Zurich , Switzerland","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5065613057","display_name":"Yiqu Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Yang, Yiqu","raw_affiliation_strings":["ETH Zurich Zurich , Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich Zurich , Switzerland","institution_ids":["https://openalex.org/I35440088"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I35440088"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5741999745368958,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.5741999745368958,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.19169999659061432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.02160000056028366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multidimensional-item-response-theory","display_name":"multidimensional item response theory","score":0.8700000047683716},{"id":"https://openalex.org/keywords/spectral-approximation","display_name":"spectral approximation","score":0.5600000023841858}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8447999954223633},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.694599986076355},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6567000150680542},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.5819000005722046},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.5295000076293945},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.5001000165939331},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4893999993801117},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.4537999927997589},{"id":"https://openalex.org/C44648626","wikidata":"https://www.wikidata.org/wiki/Q1049848","display_name":"Percentage point","level":2,"score":0.45010000467300415},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.43540000915527344},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.36910000443458557},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.3416999876499176},{"id":"https://openalex.org/C181447626","wikidata":"https://www.wikidata.org/wiki/Q1224359","display_name":"Differential item functioning","level":4,"score":0.3312000036239624},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.3052999973297119},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C63002673","wikidata":"https://www.wikidata.org/wiki/Q2260590","display_name":"Scoring rule","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.28700000047683716},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28189998865127563},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2782000005245209},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2632000148296356},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2623000144958496}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.00482","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.00482","pdf_url":"https://arxiv.org/pdf/2609.00482","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.00482","doi":"https://doi.org/10.48550/arxiv.2609.00482","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.00482","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.00482","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.00482","pdf_url":"https://arxiv.org/pdf/2609.00482","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7206170481.pdf","grobid_xml":"https://content.openalex.org/works/W7206170481.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W1872197921","https://openalex.org/W1979354511","https://openalex.org/W2013223495","https://openalex.org/W2053154970","https://openalex.org/W2078483536","https://openalex.org/W2084413241","https://openalex.org/W2117756735","https://openalex.org/W3123436326","https://openalex.org/W4392181972","https://openalex.org/W4399759481","https://openalex.org/W4414809385","https://openalex.org/W4415087836","https://openalex.org/W4416035026","https://openalex.org/W7119535842","https://openalex.org/W7140802954","https://openalex.org/W7154463862","https://openalex.org/W7162802310","https://openalex.org/W7162988157","https://openalex.org/W7169611917","https://openalex.org/W7169838747"],"related_works":["https://openalex.org/W4391591687","https://openalex.org/W7210269356","https://openalex.org/W7172149856","https://openalex.org/W7204187820","https://openalex.org/W4415328433","https://openalex.org/W7160607605","https://openalex.org/W7214754780","https://openalex.org/W7223149495","https://openalex.org/W7169838747","https://openalex.org/W7138918449"],"abstract_inverted_index":{"Small":[0],"leaderboard":[1,188],"gaps":[2,189],"are":[3,25,66],"often":[4],"interpreted":[5],"as":[6],"evidence":[7,194],"that":[8,78,195],"one":[9,49,123],"language":[10],"model":[11,61],"is":[12,199],"better":[13],"than":[14],"another,":[15],"but":[16],"their":[17,129],"sign":[18],"may":[19],"depend":[20],"on":[21],"which":[22],"benchmark":[23,140,184],"items":[24,53,65],"included.":[26],"We":[27],"test":[28],"this":[29],"using":[30],"item-level":[31],"responses":[32],"from":[33],"five":[34,115],"benchmarks":[35],"and":[36,87,104,155,186],"a":[37,95,167],"family-label-free":[38],"spectral":[39],"approximation":[40],"to":[41,68,100,183],"multidimensional":[42],"item-response":[43],"theory":[44],"(MIRT).":[45],"In":[46],"owner-disjoint":[47],"folds,":[48],"owner":[50,161],"half":[51,74],"identifies":[52],"with":[54,75],"low":[55],"residual":[56,156],"differential":[57],"item":[58,85,101],"functioning":[59],"across":[60,83,160,170],"families":[62],"(low-DIF).":[63],"These":[64],"used":[67],"score":[69],"models":[70],"in":[71,112],"the":[72,80,196],"other":[73],"frozen":[76],"weights":[77],"preserve":[79],"benchmark's":[81],"composition":[82],"metadata-defined":[84],"groups":[86],"easiness":[88],"strata.":[89],"Equally":[90],"short":[91],"matched-random":[92,130],"subtests":[93],"provide":[94],"baseline":[96],"for":[97],"variation":[98],"due":[99],"subsampling.":[102],"Full-benchmark":[103],"low-DIF":[105],"rankings":[106,175],"remain":[107],"strongly":[108],"correlated":[109],"($\u03c4_b=.900$--$.948$).":[110],"Yet":[111],"four":[113],"of":[114,118],"benchmarks,":[116],"30.9--47.1\\%":[117],"cross-family":[119],"pairs":[120],"initially":[121],"within":[122],"percentage":[124,134],"point":[125],"reverse":[126],"order,":[127],"exceeding":[128],"medians":[131],"by":[132,193],"16.9--28.6":[133],"points":[135],"(all":[136],"$p=.001$).":[137],"The":[138,148],"fifth":[139],"shows":[141,166],"no":[142,164],"reliable":[143],"excess":[144],"($-0.9$":[145],"points,":[146],"$p=.689$).":[147],"pattern":[149],"survives":[150],"all":[151],"pre-specified":[152],"population":[153],"perturbations,":[154],"item--family":[157],"signatures":[158],"replicate":[159],"halves;":[162],"however,":[163],"family":[165],"consistent":[168],"advantage":[169],"benchmarks.":[171],"Thus,":[172],"globally":[173],"stable":[174],"can":[176],"still":[177],"leave":[178],"individual":[179],"near-tie":[180],"orderings":[181],"sensitive":[182],"composition,":[185],"sub-one-point":[187],"should":[190],"be":[191],"accompanied":[192],"implied":[197],"ordering":[198],"composition-robust.":[200]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-03T00:00:00"}