{"id":"https://openalex.org/W7161031181","doi":"https://doi.org/10.48550/arxiv.2605.11209","title":"Measuring Five-Nines Reliability: Sample-Efficient LLM Evaluation in Saturated Benchmarks","display_name":"Measuring Five-Nines Reliability: Sample-Efficient LLM Evaluation in Saturated Benchmarks","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7161031181","doi":"https://doi.org/10.48550/arxiv.2605.11209"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.11209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.11209","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050355780","display_name":"Eungyeup Kim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Eungyeup","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052285414","display_name":"Chenchen Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Chenchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107853035","display_name":"V. Tiwari","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tiwari, Vashisth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136069131","display_name":"J. Zico Kolter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kolter, J. Zico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14190000295639038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.14190000295639038,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1306000053882599,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.1265999972820282,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6488999724388123},{"id":"https://openalex.org/keywords/parameterized-complexity","display_name":"Parameterized complexity","score":0.6126999855041504},{"id":"https://openalex.org/keywords/monte-carlo-method","display_name":"Monte Carlo method","score":0.5078999996185303},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"Importance sampling","score":0.48489999771118164},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.45989999175071716},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.43140000104904175},{"id":"https://openalex.org/keywords/coverage-probability","display_name":"Coverage probability","score":0.34549999237060547},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.34360000491142273}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6593999862670898},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6488999724388123},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.6126999855041504},{"id":"https://openalex.org/C19499675","wikidata":"https://www.wikidata.org/wiki/Q232207","display_name":"Monte Carlo method","level":2,"score":0.5078999996185303},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.48489999771118164},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.45989999175071716},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.44850000739097595},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.43140000104904175},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3781000077724457},{"id":"https://openalex.org/C2776292839","wikidata":"https://www.wikidata.org/wiki/Q5179217","display_name":"Coverage probability","level":3,"score":0.34549999237060547},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.34150001406669617},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.33629998564720154},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32670000195503235},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3239000141620636},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C2777317252","wikidata":"https://www.wikidata.org/wiki/Q18393516","display_name":"Rare events","level":2,"score":0.2913999855518341},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.29030001163482666},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.28540000319480896},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.27309998869895935},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C32230216","wikidata":"https://www.wikidata.org/wiki/Q7882499","display_name":"Uncertainty quantification","level":2,"score":0.2621999979019165},{"id":"https://openalex.org/C70518039","wikidata":"https://www.wikidata.org/wiki/Q16000077","display_name":"Dimensionality reduction","level":2,"score":0.2615000009536743}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.11209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.11209","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11209","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0],"existing":[1,219],"benchmarks":[2,178],"demonstrate":[3],"the":[4,20,113,132,204],"near-perfect":[5],"performance":[6],"of":[7,25,108,115,195,206,216],"large":[8,74],"language":[9],"models":[10,172],"(LLMs)":[11],"on":[12,128,140,176],"various":[13],"tasks,":[14],"this":[15,46,89,118],"apparent":[16],"saturation":[17],"often":[18],"obscures":[19],"need":[21],"for":[22,112,221],"rigorous":[23],"evaluation":[24,82,205,217],"their":[26,222],"reliability.":[27],"In":[28,88],"real-world":[29],"deployment,":[30],"however,":[31],"achieving":[32],"extremely":[33],"high":[34],"reliability":[35,188,208],"(e.g.,":[36],"\"five-nines\"":[37],"(99.999%)":[38],"vs.":[39],"\"three-nines\"":[40],"(99.9%))":[41],"is":[42,56,189],"fundamentally":[43],"critical,":[44],"as":[45],"gap":[47],"results":[48],"in":[49,53,58,160,182,209,225],"an":[50],"order-of-magnitude":[51],"increase":[52],"failures,":[54],"which":[55],"catastrophic":[57],"reliability-critical":[59],"applications.":[60,227],"Still,":[61],"estimating":[62],"such":[63],"a":[64,105,124,190,211],"rare":[65],"failure":[66,184],"probability":[67],"with":[68,173],"tight":[69],"confidence":[70],"bounds":[71],"requires":[72],"prohibitively":[73],"LLM":[75,94],"inference":[76],"sizes,":[77],"making":[78],"standard":[79,177],"Monte":[80],"Carlo":[81],"infeasible":[83],"under":[84],"limited":[85],"compute":[86],"budgets.":[87],"paper,":[90],"we":[91,120],"observe":[92],"that":[93,171,187],"failures":[95],"exhibit":[96],"strong":[97],"systematic":[98],"patterns:":[99],"across":[100,150],"broad":[101],"parameterized":[102,151],"input":[103],"spaces,":[104],"small":[106],"subset":[107],"inputs":[109,130],"disproportionately":[110],"accounts":[111],"majority":[114],"failures.":[116],"Leveraging":[117],"observation,":[119],"propose":[121],"to":[122,157,164],"learn":[123],"sampling":[125],"distribution":[126],"concentrated":[127],"failure-prone":[129],"via":[131],"cross-entropy":[133],"method":[134],"(CEM).":[135],"We":[136],"evaluate":[137],"our":[138],"framework":[139,202],"three":[141],"LLMs,":[142,210],"Qwen2.5-Math-7B-Instruct,":[143],"gpt-oss-20b-low,":[144],"and":[145,154,192,213],"Gemini":[146],"2.5":[147],"Flash":[148],"Lite,":[149],"GSM8K":[152],"templates":[153],"achieve":[155],"up":[156],"156.22x":[158],"reduction":[159],"required":[161],"inferences":[162],"compared":[163],"naive":[165],"uniform":[166],"sampling.":[167],"Our":[168,198],"estimates":[169],"reveal":[170],"indistinguishable":[174],"accuracy":[175],"can":[179],"differ":[180],"substantially":[181],"estimated":[183],"rates,":[185],"underscoring":[186],"distinct":[191,212],"measurable":[193],"axis":[194],"model":[196],"quality.":[197],"simple":[199],"yet":[200],"practical":[201],"enables":[203],"extreme":[207],"underexplored":[214],"dimension":[215],"beyond":[218],"benchmarks,":[220],"growing":[223],"use":[224],"reliability-sensitive":[226]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-14T00:00:00"}