{"id":"https://openalex.org/W7208759730","doi":"https://doi.org/10.48550/arxiv.2609.02899","title":"Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards","display_name":"Contamination Inflates Scores but Rarely Reorders Large Language Model Leaderboards","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7208759730","doi":"https://doi.org/10.48550/arxiv.2609.02899"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.02899","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02899","pdf_url":"https://arxiv.org/pdf/2609.02899","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.02899","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013483002","display_name":"Xingyao Xiao","orcid":"https://orcid.org/0000-0001-8430-0438","observed_orcids":["https://orcid.org/0000-0001-8430-0438"]},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiao, Xingyao","raw_affiliation_strings":["Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004542060","display_name":"Yihong Cheng","orcid":"https://orcid.org/0000-0002-1763-7345","observed_orcids":["https://orcid.org/0000-0002-1763-7345"]},"institutions":[{"id":"https://openalex.org/I6469544","display_name":"City University of Macau","ror":"https://ror.org/04gpd4q15","country_code":"MO","type":"education","lineage":["https://openalex.org/I6469544"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"Cheng, Yihong","raw_affiliation_strings":["City University of Macau"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"City University of Macau","institution_ids":["https://openalex.org/I6469544"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6439999938011169,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6439999938011169,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.08940000087022781,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.08709999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark-contamination","display_name":"benchmark contamination","score":0.9940000176429749},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8349999785423279},{"id":"https://openalex.org/keywords/score-inflation","display_name":"score inflation","score":0.6200000047683716},{"id":"https://openalex.org/keywords/benchmark-reliability","display_name":"benchmark reliability","score":0.546999990940094}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6578999757766724},{"id":"https://openalex.org/C181447626","wikidata":"https://www.wikidata.org/wiki/Q1224359","display_name":"Differential item functioning","level":4,"score":0.6082000136375427},{"id":"https://openalex.org/C112570922","wikidata":"https://www.wikidata.org/wiki/Q60528603","display_name":"Contamination","level":2,"score":0.5683000087738037},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5595999956130981},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.5005000233650208},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4950000047683716},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4819999933242798},{"id":"https://openalex.org/C93226319","wikidata":"https://www.wikidata.org/wiki/Q193137","display_name":"Differential (mechanical device)","level":2,"score":0.43209999799728394},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.4212999939918518},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.39750000834465027},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.39250001311302185},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3529999852180481},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.3303999900817871},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3156999945640564},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.31529998779296875},{"id":"https://openalex.org/C126780896","wikidata":"https://www.wikidata.org/wiki/Q899871","display_name":"Distortion (music)","level":4,"score":0.2939999997615814},{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.28209999203681946},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27730000019073486},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.2768000066280365},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.2766000032424927},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26759999990463257},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26489999890327454},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.2648000121116638}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.02899","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02899","pdf_url":"https://arxiv.org/pdf/2609.02899","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.02899","doi":"https://doi.org/10.48550/arxiv.2609.02899","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.02899","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.02899","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02899","pdf_url":"https://arxiv.org/pdf/2609.02899","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7208759730.pdf","grobid_xml":"https://content.openalex.org/works/W7208759730.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4416368190","https://openalex.org/W4391591687","https://openalex.org/W4404782840","https://openalex.org/W7165626289","https://openalex.org/W7138934342","https://openalex.org/W7140229896","https://openalex.org/W7166876690","https://openalex.org/W4404783013","https://openalex.org/W4414849625","https://openalex.org/W4399115495"],"abstract_inverted_index":{"Benchmark":[0],"contamination,":[1,101],"the":[2,17,43,60,75,112,141,151,170,177,212,218],"leakage":[3],"of":[4,19,45,53,63,100,125,186,221],"test":[5],"items":[6],"into":[7],"training":[8,143],"data,":[9],"is":[10,162,174,202],"widely":[11],"described":[12],"as":[13,50,231],"a":[14,51,70,97,135,155,159,165,226,232],"threat":[15],"to":[16,180],"reliability":[18],"large":[20],"language":[21],"model":[22,137],"(LLM)":[23],"leaderboards.":[24],"We":[25,47,146,224],"argue":[26],"that":[27,73,169,237],"this":[28],"concern":[29],"conflates":[30],"two":[31,195],"distinct":[32],"questions:":[33],"whether":[34,40],"contamination":[35,49,120,173,192],"inflates":[36,207],"absolute":[37,208],"scores,":[38],"and":[39,56,79,92,132,158,164,214,235],"it":[41,58,117,206],"reorders":[42],"ranking":[44,215],"models.":[46],"recast":[48],"violation":[52],"anchor-item":[54],"invariance":[55,228],"measure":[57,113],"through":[59],"differential":[61,172,191,222],"functioning":[62],"original":[64],"versus":[65],"semantically":[66],"equivalent":[67],"paraphrased":[68],"items,":[69],"within-item":[71],"contrast":[72],"holds":[74],"measured":[76],"skill":[77],"fixed":[78],"isolates":[80],"memorization":[81],"from":[82,87],"capability.":[83],"Using":[84],"per-instance":[85],"responses":[86],"47":[88],"publicly":[89],"released":[90,230],"models":[91,94,201],"74":[93],"finetuned":[95],"with":[96,183],"known":[98],"dose":[99],"across":[102,194],"four":[103],"benchmarks":[104],"(ARC,":[105],"GSM8K,":[106],"HellaSwag,":[107],"MMLU),":[108],"we":[109],"first":[110],"calibrate":[111],"against":[114],"ground":[115],"truth:":[116],"recovers":[118],"injected":[119],"dose-responsively":[121],"(a":[122],"corrected":[123],"effect":[124],"+0.187":[126],"accuracy":[127],"points":[128],"for":[129],"test-set":[130],"leakage)":[131],"never":[133],"flags":[134],"negative-control":[136],"trained":[138],"only":[139,184],"on":[140],"legitimate":[142],"split":[144],"(-0.012).":[145],"then":[147],"quantify":[148],"leaderboard":[149,157,161],"impact:":[150],"rank":[152],"correlation":[153],"between":[154],"standard":[156],"paraphrase-controlled":[160,240],"0.997,":[163],"sensitivity":[166],"analysis":[167],"shows":[168],"observed":[171],"far":[175],"below":[176],"level":[178],"needed":[179],"move":[181],"rankings,":[182],"3":[185],"188":[187],"model-by-benchmark":[188],"cases":[189],"showing":[190],"corroborated":[193],"references.":[196],"Contamination":[197],"among":[198],"these":[199],"public":[200],"therefore":[203],"largely":[204],"uniform:":[205],"scores":[209],"without":[210],"reordering":[211],"leaderboard,":[213],"distortion":[216],"requires":[217],"rare":[219],"case":[220],"contamination.":[223],"provide":[225],"calibrated":[227],"audit,":[229],"reference":[233],"implementation,":[234],"recommend":[236],"leaderboards":[238],"report":[239],"rankings":[241],"alongside":[242],"confidence":[243],"intervals.":[244]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-05T00:00:00"}