{"id":"https://openalex.org/W7214182159","doi":"https://doi.org/10.48550/arxiv.2609.28177","title":"How Sensitive Are LLM Leaderboard Claims to Hidden Model Selection?","display_name":"How Sensitive Are LLM Leaderboard Claims to Hidden Model Selection?","publication_year":2026,"publication_date":"2026-09-23","ids":{"openalex":"https://openalex.org/W7214182159","doi":"https://doi.org/10.48550/arxiv.2609.28177"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.28177","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.28177","pdf_url":"https://arxiv.org/pdf/2609.28177","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.28177","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5152618774","display_name":"Chen Yang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I91045830","display_name":"Texas A&M University","ror":"https://ror.org/01f5ytq51","country_code":"US","type":"education","lineage":["https://openalex.org/I91045830"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Chen","raw_affiliation_strings":["Department of Statistics , Texas A&M University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics , Texas A&M University","institution_ids":["https://openalex.org/I91045830"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153428096","display_name":"Jun Chen","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Jun","raw_affiliation_strings":["Division of Computational Biology ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Division of Computational Biology ,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.20720000565052032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.20720000565052032,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10467","display_name":"Psychometric Methodologies and Testing","score":0.08209999650716782,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13141","display_name":"Statistical Methods and Applications","score":0.07069999724626541,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8140000104904175},{"id":"https://openalex.org/keywords/model-selection","display_name":"model selection","score":0.6449999809265137},{"id":"https://openalex.org/keywords/sensitivity-analysis","display_name":"sensitivity analysis","score":0.5540000200271606},{"id":"https://openalex.org/keywords/mmlu","display_name":"MMLU","score":0.5720000267028809}],"concepts":[{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.661899983882904},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6394000053405762},{"id":"https://openalex.org/C150921843","wikidata":"https://www.wikidata.org/wiki/Q1170431","display_name":"Resampling","level":2,"score":0.6148999929428101},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.536300003528595},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.5170000195503235},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4968999922275543},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.47850000858306885},{"id":"https://openalex.org/C93959086","wikidata":"https://www.wikidata.org/wiki/Q6888345","display_name":"Model selection","level":2,"score":0.44780001044273376},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.40230000019073486},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.36910000443458557},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.362199991941452},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.3434000015258789},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.3391000032424927},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3379000127315521},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3149999976158142},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.31470000743865967},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3142000138759613},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.29190000891685486},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C90329073","wikidata":"https://www.wikidata.org/wiki/Q914232","display_name":"Ask price","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C122048520","wikidata":"https://www.wikidata.org/wiki/Q2913954","display_name":"Percentile","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.2703999876976013},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.26499998569488525},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2599000036716461},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.2515999972820282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.28177","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.28177","pdf_url":"https://arxiv.org/pdf/2609.28177","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.28177","doi":"https://doi.org/10.48550/arxiv.2609.28177","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.28177","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.28177","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.28177","pdf_url":"https://arxiv.org/pdf/2609.28177","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214182159.pdf","grobid_xml":"https://content.openalex.org/works/W7214182159.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W1555759181","https://openalex.org/W1798749056","https://openalex.org/W2006546769","https://openalex.org/W2173631609","https://openalex.org/W2532914530","https://openalex.org/W2624735416","https://openalex.org/W4401499842","https://openalex.org/W4404345312","https://openalex.org/W4406975857","https://openalex.org/W4415798641","https://openalex.org/W4416045832","https://openalex.org/W4417136306","https://openalex.org/W4417172194","https://openalex.org/W6966828174","https://openalex.org/W7116918102","https://openalex.org/W7154217415","https://openalex.org/W7154395918","https://openalex.org/W7160632553","https://openalex.org/W7162802310","https://openalex.org/W7170077506"],"related_works":["https://openalex.org/W4391591687","https://openalex.org/W7160607605","https://openalex.org/W7163777991","https://openalex.org/W7220954687","https://openalex.org/W7208759730","https://openalex.org/W7214646924","https://openalex.org/W7203745494","https://openalex.org/W7157939549","https://openalex.org/W7167228722","https://openalex.org/W7212033163"],"abstract_inverted_index":{"LLM":[0,123],"leaderboard":[1],"gains":[2],"can":[3,31,144],"reflect":[4],"selection":[5],"among":[6],"privately":[7],"evaluated":[8],"model":[9],"variants,":[10],"yet":[11],"neither":[12],"the":[13,80,86,121,140,149,162],"number":[14],"of":[15,37,68,116],"variants":[16,27],"nor":[17],"their":[18],"dependence":[19],"is":[20,96,101],"public.":[21],"We":[22],"ask":[23],"how":[24],"many":[25],"hidden":[26,150],"a":[28,38,42,46,51,57,66,69,90],"published":[29],"margin":[30,53],"support":[32,130],"while":[33],"retaining":[34],"statistical":[35,129],"evidence":[36],"provider's":[39],"advantage":[40],"over":[41],"fixed":[43,47],"comparator.":[44],"For":[45],"candidate":[48],"family":[49],"under":[50,103],"Gaussian":[52],"model,":[54],"we":[55],"derive":[56],"sensitivity":[58],"curve":[59],"that":[60,126,138],"reports":[61],"this":[62],"maximum":[63],"count":[64],"as":[65],"function":[67],"lower":[70],"bound":[71],"on":[72,120,146],"within-family":[73],"correlation.":[74,152],"The":[75,153],"relevant":[76],"correlation":[77,95,100],"must":[78],"match":[79],"score":[81],"used":[82],"for":[83,134],"ranking":[84],"and":[85,106],"sampling":[87],"model:":[88],"in":[89],"controlled":[91],"family,":[92],"pooled":[93],"item":[94,104],"0.90,":[97],"whereas":[98],"composite-score":[99],"0.46":[102],"resampling":[105],"0.92":[107],"when":[108],"MMLU":[109],"subjects":[110],"are":[111],"resampled.":[112],"An":[113],"item-based":[114],"audit":[115],"394":[117],"adjacent-rank":[118],"claims":[119,137],"Open":[122],"Leaderboard":[124],"finds":[125],"391":[127],"lack":[128],"even":[131],"before":[132],"accounting":[133],"selection.":[135],"Among":[136],"pass":[139],"uncorrected":[141],"test,":[142],"certification":[143],"depend":[145],"assumptions":[147,158],"about":[148],"family's":[151],"resulting":[154],"curves":[155],"make":[156],"these":[157],"explicit":[159],"without":[160],"estimating":[161],"unobserved":[163],"search":[164],"size.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-25T00:00:00"}