{"id":"https://openalex.org/W7138511917","doi":"https://doi.org/10.48550/arxiv.2603.13285","title":"Brittlebench: Quantifying LLM robustness via prompt sensitivity","display_name":"Brittlebench: Quantifying LLM robustness via prompt sensitivity","publication_year":2026,"publication_date":"2026-02-27","ids":{"openalex":"https://openalex.org/W7138511917","doi":"https://doi.org/10.48550/arxiv.2603.13285"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.13285","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13285","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.13285","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113161539","display_name":"Angelika Romanou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Romanou, Angelika","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121657673","display_name":"Mark Ibrahim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ibrahim, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082026462","display_name":"Candace Ross","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ross, Candace","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129662272","display_name":"Chantal Shaib","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shaib, Chantal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129679590","display_name":"Kerem Okta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Oktar, Kerem","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129731155","display_name":"Sam Bell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bell, Samuel J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129686620","display_name":"Elia Ovalle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ovalle, Anaelia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008013895","display_name":"Jesse Dodge","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dodge, Jesse","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129720320","display_name":"Antoine Bosselut","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bosselut, Antoine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109576489","display_name":"Koustuv Sinha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sinha, Koustuv","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5062696000","display_name":"Adina Williams","orcid":"https://orcid.org/0000-0001-5281-3343"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Williams, Adina","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.46399998664855957,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.46399998664855957,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.31119999289512634,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.05950000137090683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7461000084877014},{"id":"https://openalex.org/keywords/sensitivity","display_name":"Sensitivity (control systems)","score":0.6496000289916992},{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.625},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.5375000238418579},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.5037000179290771},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.49939998984336853},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.4156999886035919},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.36250001192092896}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7461000084877014},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6807000041007996},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.6496000289916992},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.625},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.5375000238418579},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.5037000179290771},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.49939998984336853},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4156999886035919},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3874000012874603},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.36250001192092896},{"id":"https://openalex.org/C169345407","wikidata":"https://www.wikidata.org/wiki/Q8216221","display_name":"Uncorrelated","level":2,"score":0.33550000190734863},{"id":"https://openalex.org/C167928553","wikidata":"https://www.wikidata.org/wiki/Q1376021","display_name":"Estimation theory","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.325300008058548},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.31209999322891235},{"id":"https://openalex.org/C108311543","wikidata":"https://www.wikidata.org/wiki/Q7915756","display_name":"Variance-based sensitivity analysis","level":4,"score":0.3100000023841858},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2996000051498413},{"id":"https://openalex.org/C2986577269","wikidata":"https://www.wikidata.org/wiki/Q11306265","display_name":"Random noise","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2919999957084656},{"id":"https://openalex.org/C2983447341","wikidata":"https://www.wikidata.org/wiki/Q1413083","display_name":"Model parameter","level":2,"score":0.2915000021457672},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C3018076075","wikidata":"https://www.wikidata.org/wiki/Q1826427","display_name":"Variance components","level":2,"score":0.260699987411499},{"id":"https://openalex.org/C3020402766","wikidata":"https://www.wikidata.org/wiki/Q104376712","display_name":"Prior information","level":2,"score":0.25189998745918274}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.13285","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13285","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.13285","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.13285","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"evaluation":[1,87],"methods":[2],"largely":[3],"rely":[4],"on":[5],"clean,":[6],"static":[7],"benchmarks,":[8,107],"which":[9,35],"can":[10,36,70,164],"overestimate":[11],"true":[12,31],"model":[13,62,110,145,194],"performance":[14,111,172],"by":[15],"failing":[16],"to":[17,64,73,90,102,112,168,191],"capture":[18],"the":[19,49,93,132,148,171,180],"noise":[20],"and":[21,108,155,186,188],"variability":[22],"inherent":[23],"in":[24,137],"real-world":[25],"user":[26],"inputs.":[27],"This":[28],"is":[29],"especially":[30],"for":[32,60,166,174,182],"language":[33],"models,":[34,157,187],"face":[37],"human-generated":[38],"text":[39],"queries":[40],"containing":[41],"mistakes,":[42],"typos,":[43],"or":[44,67],"alternative":[45],"ways":[46],"of":[47,95,105,135,139,151,170],"phrasing":[48],"same":[50],"question.":[51],"In":[52],"this":[53,81],"work,":[54],"we":[55,83,158],"introduce":[56],"a":[57,85,103,128,175],"theoretical":[58],"framework":[59],"quantifying":[61],"sensitivity":[63,94],"prompt":[65],"variants,":[66],"brittleness,":[68],"that":[69,160],"enable":[71],"us":[72,190],"disentangle":[74],"data-induced":[75],"difficulty":[76],"from":[77],"prompt-related":[78],"variability.":[79],"Using":[80],"framework,":[82],"design":[84],"novel":[86],"pipeline,":[88],"Brittlebench,":[89],"holistically":[91],"evaluate":[92],"frontier":[96],"models.":[97],"We":[98],"apply":[99],"semantics-preserving":[100,161],"perturbations":[101,120,163],"suite":[104],"popular":[106],"observe":[109],"degrade":[113],"as":[114,116],"much":[115],"12%.":[117],"However,":[118],"these":[119],"do":[121],"not":[122],"affect":[123],"all":[124],"models":[125,136],"equally:":[126],"even":[127],"single":[129],"perturbation":[130],"alters":[131],"relative":[133],"ranking":[134],"63%":[138],"cases,":[140],"impacting":[141],"conclusions":[142],"about":[143],"comparative":[144],"performance.":[146],"Decomposing":[147],"total":[149],"variance":[150,173],"both":[152],"state-of-the-art":[153],"open-weight":[154],"commercial":[156],"find":[159],"input":[162],"account":[165],"up":[167],"half":[169],"given":[176],"model.":[177],"Brittlebench":[178],"highlights":[179],"need":[181],"more":[183],"robust":[184],"evaluations":[185],"allows":[189],"systematically":[192],"understand":[193],"brittleness.":[195]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}