{"id":"https://openalex.org/W4414528631","doi":"https://doi.org/10.52202/085713-0576","title":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","display_name":"Signal and Noise: A Framework for Reducing Uncertainty in Language Model Evaluation","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4414528631","doi":"https://doi.org/10.52202/085713-0576"},"language":"en","primary_location":{"id":"doi:10.52202/085713-0576","doi":"https://doi.org/10.52202/085713-0576","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0576","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2508.13144","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5115024195","display_name":"David Heineman","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Heineman","raw_affiliation_strings":["Allen Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014923355","display_name":"Valentin Hofmann","orcid":"https://orcid.org/0000-0001-6603-3428","observed_orcids":["https://orcid.org/0000-0001-6603-3428"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valentin Hofmann","raw_affiliation_strings":["Allen Institute for AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074637993","display_name":"Ian Magnusson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ian Magnusson","raw_affiliation_strings":["Allen Institute for AI; University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI; University of Washington","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029619813","display_name":"Yuling Gu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuling Gu","raw_affiliation_strings":["New York University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088517824","display_name":"Noah A. Smith","orcid":"https://orcid.org/0000-0002-2310-6380","observed_orcids":["https://orcid.org/0000-0002-2310-6380"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Noah Smith","raw_affiliation_strings":["Allen Institute for AI, University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, University of Washington","institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082305994","display_name":"Hannaneh Hajishirzi","orcid":"https://orcid.org/0000-0002-1055-6657","observed_orcids":["https://orcid.org/0000-0002-1055-6657"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanna Hajishirzi","raw_affiliation_strings":["University of Washington/AI2"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Washington/AI2","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066588555","display_name":"Kyle Lo","orcid":"https://orcid.org/0000-0002-1804-2853","observed_orcids":["https://orcid.org/0000-0002-1804-2853"]},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kyle Lo","raw_affiliation_strings":["Allen Institute for Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5008013895","display_name":"Jesse Dodge","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jesse Dodge","raw_affiliation_strings":["Meta (FAIR)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meta (FAIR)","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.992,"has_fulltext":true,"cited_by_count":19,"citation_normalized_percentile":{"value":0.97656116,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"19286","last_page":"19327"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7997999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7997999787330627,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.17430000007152557,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.003599999938160181,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-evaluation","display_name":"LLM evaluation","score":0.8339999914169312},{"id":"https://openalex.org/keywords/signal-to-noise-ratio","display_name":"signal-to-noise ratio","score":0.9750000238418579},{"id":"https://openalex.org/keywords/multi-task-evaluation","display_name":"multi-task evaluation","score":0.628000020980835}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7523999810218811},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7117999792098999},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7026000022888184},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.6319000124931335},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.4975999891757965},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.48910000920295715},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.47269999980926514},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4702000021934509},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.4668999910354614},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4652999937534332},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.46299999952316284},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.4602000117301941},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4377000033855438},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3684000074863434},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.33709999918937683},{"id":"https://openalex.org/C29265498","wikidata":"https://www.wikidata.org/wiki/Q7047719","display_name":"Noise measurement","level":3,"score":0.3294000029563904},{"id":"https://openalex.org/C104267543","wikidata":"https://www.wikidata.org/wiki/Q208163","display_name":"Signal processing","level":3,"score":0.32019999623298645},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C13944312","wikidata":"https://www.wikidata.org/wiki/Q7512748","display_name":"Signal-to-noise ratio (imaging)","level":2,"score":0.29159998893737793},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.2842999994754791},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26759999990463257}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-0576","doi":"https://doi.org/10.52202/085713-0576","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0576","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.13144","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.13144","pdf_url":"https://arxiv.org/pdf/2508.13144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2508.13144","doi":"https://doi.org/10.48550/arxiv.2508.13144","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.13144","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2508.13144","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.13144","pdf_url":"https://arxiv.org/pdf/2508.13144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G5770373782","display_name":"The CSGrad4US Fellowship Program","funder_award_id":"2313998","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414528631.pdf","grobid_xml":"https://content.openalex.org/works/W4414528631.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W2794325560","https://openalex.org/W3001279689","https://openalex.org/W3005700362","https://openalex.org/W3083410900","https://openalex.org/W3100501376","https://openalex.org/W3118781290","https://openalex.org/W3177813494","https://openalex.org/W3195140297","https://openalex.org/W3210904070","https://openalex.org/W4281690148","https://openalex.org/W4283026156","https://openalex.org/W4320003957","https://openalex.org/W4387799916","https://openalex.org/W4388926373","https://openalex.org/W4392181972","https://openalex.org/W4392824982","https://openalex.org/W4394868329","https://openalex.org/W4399448194","https://openalex.org/W4399455140","https://openalex.org/W4399510412","https://openalex.org/W4399759481","https://openalex.org/W4399911984","https://openalex.org/W4400342789","https://openalex.org/W4400611750","https://openalex.org/W4401042689","https://openalex.org/W4401306886","https://openalex.org/W4403324134","https://openalex.org/W4403575412","https://openalex.org/W4404345312","https://openalex.org/W4404781183","https://openalex.org/W4404987046","https://openalex.org/W4405095648","https://openalex.org/W4406032137","https://openalex.org/W4406072757","https://openalex.org/W4407231846","https://openalex.org/W4407633227","https://openalex.org/W4411120158","https://openalex.org/W4412888025","https://openalex.org/W4415797383","https://openalex.org/W4415797507","https://openalex.org/W4415974127","https://openalex.org/W4416045832","https://openalex.org/W7133211296","https://openalex.org/W7133225605"],"related_works":["https://openalex.org/W4386150511","https://openalex.org/W7221873429","https://openalex.org/W4416050957","https://openalex.org/W4417115503","https://openalex.org/W4403662613","https://openalex.org/W4398879428","https://openalex.org/W6947952751","https://openalex.org/W7116184612","https://openalex.org/W7160309245","https://openalex.org/W4399759481"],"abstract_inverted_index":{"Developing":[0],"large":[1],"language":[2,232],"models":[3,62,233],"is":[4],"expensive":[5],"and":[6,37,66,95,145,156,219,229],"involves":[7],"making":[8,90],"decisions":[9,91],"with":[10,81,97],"small":[11,93],"experiments,":[12,228],"typically":[13],"by":[14,201],"evaluating":[15],"on":[16],"large,":[17],"multi-task":[18,178],"evaluation":[19,42,248],"suites.":[20],"In":[21],"this":[22],"work,":[23],"we":[24,121,134],"analyze":[25],"specific":[26],"properties":[27],"which":[28,210],"make":[29],"a":[30,56,68,82,139,188,241],"benchmark":[31,249],"more":[32,87,117,176],"reliable":[33,88,177],"for":[34,216,226],"such":[35],"decisions,":[36],"interventions":[38,124],"to":[39,59,71,116,126,138,153,168,175,192,196,213,236],"design":[40],"higher-quality":[41],"benchmarks.":[43],"We":[44,77,161,180,199,222],"introduce":[45,122],"two":[46],"key":[47],"metrics":[48],"that":[49,79,109,136,141,164,183,203],"show":[50],"differences":[51],"in":[52,240],"current":[53],"benchmarks:":[54],"signal,":[55],"benchmark's":[57,69],"ability":[58],"separate":[60],"better":[61,83,143,154],"from":[63,234],"worse":[64],"models,":[65],"noise,":[67],"sensitivity":[70],"random":[72],"variability":[73],"between":[74],"training":[75],"steps.":[76],"demonstrate":[78],"benchmarks":[80,212,225],"signal-to-noise":[84,172],"ratio":[85],"are":[86],"when":[89],"at":[92],"scale,":[94],"those":[96,204],"less":[98],"noise":[99,113,146,194],"have":[100],"lower":[101],"scaling":[102,158],"law":[103,159],"prediction":[104],"error.":[105,160],"These":[106],"results":[107],"suggest":[108],"improving":[110],"signal":[111,129,144,218],"or":[112,130,208],"will":[114],"lead":[115],"useful":[118],"benchmarks,":[119,207],"so":[120],"three":[123],"designed":[125],"directly":[127],"affect":[128],"noise.":[131,221],"For":[132],"example,":[133],"propose":[135],"switching":[137],"metric":[140],"has":[142],"(e.g.,":[147],"perplexity":[148],"rather":[149],"than":[150],"accuracy)":[151],"leads":[152,174,195],"reliability":[155],"improved":[157],"also":[162,181],"find":[163,182],"filtering":[165],"noisy":[166],"subtasks,":[167],"improve":[169],"an":[170],"aggregate":[171],"ratio,":[173],"evaluations.":[179],"averaging":[184],"the":[185],"output":[186],"of":[187,246],"model's":[189],"intermediate":[190],"checkpoints":[191],"reduce":[193],"consistent":[197],"improvements.":[198],"conclude":[200],"recommending":[202],"creating":[205],"new":[206],"selecting":[209],"existing":[211],"use,":[214],"aim":[215],"high":[217],"low":[220],"use":[223],"30":[224],"these":[227],"375":[230],"open-weight":[231],"60M":[235],"32B":[237],"parameters,":[238],"resulting":[239],"new,":[242],"publicly":[243],"available":[244],"dataset":[245],"900K":[247],"results,":[250],"totaling":[251],"200M":[252],"instances.":[253]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":6}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2025-10-10T00:00:00"}