{"id":"https://openalex.org/W7128702410","doi":"https://doi.org/10.48550/arxiv.2602.10144","title":"When LLMs get significantly worse: A statistical approach to detect model degradations","display_name":"When LLMs get significantly worse: A statistical approach to detect model degradations","publication_year":2026,"publication_date":"2026-02-09","ids":{"openalex":"https://openalex.org/W7128702410","doi":"https://doi.org/10.48550/arxiv.2602.10144"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2602.10144","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.10144","pdf_url":"https://arxiv.org/pdf/2602.10144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7128702410"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2602.10144","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125750566","display_name":"Jonas K\u00fcbler","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"K\u00fcbler, Jonas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060824941","display_name":"Kailash Budhathoki","orcid":"https://orcid.org/0000-0002-5255-8642","observed_orcids":["https://orcid.org/0000-0002-5255-8642"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Budhathoki, Kailash","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006408039","display_name":"Matth\u00e4us Kleinde\u00dfner","orcid":"https://orcid.org/0000-0002-9907-4610","observed_orcids":["https://orcid.org/0000-0002-9907-4610"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kleindessner, Matth\u00e4us","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125708812","display_name":"XIONG ZHOU","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Xiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102414338","display_name":"Junming Yin","orcid":"https://orcid.org/0000-0001-6018-7813","observed_orcids":["https://orcid.org/0000-0001-6018-7813"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yin, Junming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056299576","display_name":"Ashish Khetan","orcid":"https://orcid.org/0000-0002-8856-1595","observed_orcids":["https://orcid.org/0000-0002-8856-1595"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khetan, Ashish","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5082384108","display_name":"George Karypis","orcid":"https://orcid.org/0000-0003-2753-1437","observed_orcids":["https://orcid.org/0000-0003-2753-1437"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karypis, George","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.23109999299049377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.23109999299049377,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.21690000593662262,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2093999981880188,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-optimization","display_name":"model optimization","score":0.5},{"id":"https://openalex.org/keywords/inference-cost-reduction","display_name":"inference cost reduction","score":0.5239999890327454},{"id":"https://openalex.org/keywords/numerical-error","display_name":"numerical error","score":0.5529999732971191},{"id":"https://openalex.org/keywords/hypothesis-testing","display_name":"hypothesis testing","score":0.7110000252723694},{"id":"https://openalex.org/keywords/mcnemar-test","display_name":"McNemar test","score":0.9990000128746033},{"id":"https://openalex.org/keywords/quantization","display_name":"quantization","score":0.7570000290870667},{"id":"https://openalex.org/keywords/false-positive-control","display_name":"false positive control","score":0.4970000088214874}],"concepts":[{"id":"https://openalex.org/C2777548347","wikidata":"https://www.wikidata.org/wiki/Q5456937","display_name":"Flagging","level":2,"score":0.8032000064849854},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7057999968528748},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5764999985694885},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.5259000062942505},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.491100013256073},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.4616999924182892},{"id":"https://openalex.org/C81081738","wikidata":"https://www.wikidata.org/wiki/Q55542","display_name":"Lossless compression","level":3,"score":0.45910000801086426},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4569999873638153},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4366999864578247},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.42899999022483826},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.4090000092983246},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.40209999680519104},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.3878999948501587},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37869998812675476},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.3402000069618225},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3276999890804291},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31360000371932983},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C2779714256","wikidata":"https://www.wikidata.org/wiki/Q25305062","display_name":"Multiple Models","level":2,"score":0.28200000524520874},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2809999883174896},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C117236510","wikidata":"https://www.wikidata.org/wiki/Q7113620","display_name":"Overdispersion","level":4,"score":0.27570000290870667},{"id":"https://openalex.org/C154606282","wikidata":"https://www.wikidata.org/wiki/Q7048579","display_name":"Nominal level","level":3,"score":0.2694999873638153},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C2776836416","wikidata":"https://www.wikidata.org/wiki/Q1364844","display_name":"False alarm","level":2,"score":0.2535000145435333},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.2513999938964844}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2602.10144","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.10144","pdf_url":"https://arxiv.org/pdf/2602.10144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2602.10144","doi":"https://doi.org/10.48550/arxiv.2602.10144","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.10144","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2602.10144","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2602.10144","pdf_url":"https://arxiv.org/pdf/2602.10144","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7128702410.pdf","grobid_xml":"https://content.openalex.org/works/W7128702410.grobid-xml"},"referenced_works_count":15,"referenced_works":["https://openalex.org/W2963929190","https://openalex.org/W4306808908","https://openalex.org/W4319166707","https://openalex.org/W4364382924","https://openalex.org/W4387947619","https://openalex.org/W4388718089","https://openalex.org/W4388926373","https://openalex.org/W4395065783","https://openalex.org/W4401306886","https://openalex.org/W4404345312","https://openalex.org/W4405903187","https://openalex.org/W4414939355","https://openalex.org/W4415796274","https://openalex.org/W7075994024","https://openalex.org/W7133211296"],"related_works":["https://openalex.org/W7213496539","https://openalex.org/W4416037335","https://openalex.org/W7165423347","https://openalex.org/W7167932887","https://openalex.org/W4416010228","https://openalex.org/W7128110441","https://openalex.org/W4401042820","https://openalex.org/W7169876819","https://openalex.org/W7203700000","https://openalex.org/W7220915611"],"abstract_inverted_index":{"Minimizing":[0],"the":[1,40,96,133,143,169,184],"inference":[2],"cost":[3],"and":[4,22,177],"latency":[5],"of":[6,14,31,82,121,168,209],"foundation":[7],"models":[8],"has":[9,43],"become":[10],"a":[11,75,83,100,118,159,179],"crucial":[12,36,125],"area":[13],"research.":[15],"Optimization":[16],"approaches":[17,150],"include":[18],"theoretically":[19,59],"lossless":[20,60],"methods":[21],"others":[23],"without":[24],"accuracy":[25,77,153,207],"guarantees":[26],"like":[27],"quantization.":[28],"In":[29],"all":[30],"these":[32],"cases":[33],"it":[34,88],"is":[35,79,127],"to":[37,58,64,72,92,111,131,151,215],"ensure":[38],"that":[39,128,183,195,201],"model":[41,51,61,114,134,193],"quality":[42],"not":[44,54,191],"degraded.":[45],"However,":[46],"even":[47,57,205],"at":[48],"temperature":[49],"zero,":[50],"generations":[52],"are":[53,196],"necessarily":[55],"robust":[56],"optimizations":[62,194],"due":[63],"numerical":[65],"errors.":[66],"We":[67,98,162,199],"thus":[68],"require":[69],"statistical":[70],"tools":[71],"decide":[73],"whether":[74,87],"finite-sample":[76],"deviation":[78],"an":[80,164],"evidence":[81],"model's":[84],"degradation":[85],"or":[86],"can":[89,211],"be":[90,212],"attributed":[91,214],"(harmless)":[93],"noise":[94],"in":[95],"evaluation.":[97],"propose":[99,148],"statistically":[101],"sound":[102],"hypothesis":[103],"testing":[104],"framework":[105],"based":[106],"on":[107,136,142,166],"McNemar's":[108],"test":[109],"allowing":[110],"efficiently":[112],"detect":[113],"degradations,":[115],"while":[116,190],"guaranteeing":[117],"controlled":[119],"rate":[120],"false":[122],"positives.":[123],"The":[124],"insight":[126],"we":[129,147],"have":[130],"confront":[132],"scores":[135],"each":[137],"sample,":[138],"rather":[139,218],"than":[140,219],"aggregated":[141],"task":[144],"level.":[145],"Furthermore,":[146],"three":[149],"aggregate":[152],"estimates":[154],"across":[155],"multiple":[156],"benchmarks":[157],"into":[158],"single":[160],"decision.":[161],"provide":[163,178],"implementation":[165],"top":[167],"largely":[170],"adopted":[171],"open":[172],"source":[173],"LM":[174],"Evaluation":[175],"Harness":[176],"case":[180],"study":[181],"illustrating":[182],"method":[185],"correctly":[186],"flags":[187],"degraded":[188],"models,":[189],"flagging":[192],"provably":[197],"lossless.":[198],"find":[200],"with":[202],"our":[203],"tests":[204],"empirical":[206],"degradations":[208,217],"0.3%":[210],"confidently":[213],"actual":[216],"noise.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-11-14T11:04:58.165337","created_date":"2026-02-13T00:00:00"}
