{"id":"https://openalex.org/W7203757690","doi":"https://doi.org/10.48550/arxiv.2608.18066","title":"On the Fragility of Self-Improving Agents: Variance, Task Order, and Underspecification","display_name":"On the Fragility of Self-Improving Agents: Variance, Task Order, and Underspecification","publication_year":2026,"publication_date":"2026-08-18","ids":{"openalex":"https://openalex.org/W7203757690","doi":"https://doi.org/10.48550/arxiv.2608.18066"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.18066","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18066","pdf_url":"https://arxiv.org/pdf/2608.18066","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.18066","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140821683","display_name":"Qinyuan Ye","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ye, Qinyuan","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5144167837","display_name":"Yu Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Yu","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017627015","display_name":"Yada Pruksachatkun","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pruksachatkun, Yada","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125557677","display_name":"Jiaxin Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Jiaxin","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066791810","display_name":"Chien-Sheng Wu","orcid":"https://orcid.org/0000-0002-5598-5324","observed_orcids":["https://orcid.org/0000-0002-5598-5324"]},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wu, Chien-Sheng","raw_affiliation_strings":["Salesforce AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce AI Research","institution_ids":["https://openalex.org/I4210155268"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210155268"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.4984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.4984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.399399995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.006899999920278788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-improving-agents","display_name":"self-improving agents","score":0.9829999804496765},{"id":"https://openalex.org/keywords/instance-based-learning","display_name":"instance-based learning","score":0.7689999938011169},{"id":"https://openalex.org/keywords/agent-reliability","display_name":"agent reliability","score":0.5839999914169312},{"id":"https://openalex.org/keywords/task-order","display_name":"task order","score":0.6919999718666077},{"id":"https://openalex.org/keywords/curriculum-effects","display_name":"curriculum effects","score":0.6140000224113464},{"id":"https://openalex.org/keywords/underspecification","display_name":"underspecification","score":0.8209999799728394},{"id":"https://openalex.org/keywords/rubrics","display_name":"rubrics","score":0.5730000138282776},{"id":"https://openalex.org/keywords/human-oversight","display_name":"human oversight","score":0.6819999814033508},{"id":"https://openalex.org/keywords/multi-step-tasks","display_name":"multi-step tasks","score":0.6700000166893005}],"concepts":[{"id":"https://openalex.org/C2779631151","wikidata":"https://www.wikidata.org/wiki/Q7883767","display_name":"Underspecification","level":2,"score":0.8198999762535095},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7458000183105469},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7314000129699707},{"id":"https://openalex.org/C167927819","wikidata":"https://www.wikidata.org/wiki/Q1930567","display_name":"Shuffling","level":2,"score":0.5497000217437744},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.47699999809265137},{"id":"https://openalex.org/C80191262","wikidata":"https://www.wikidata.org/wiki/Q5477668","display_name":"Fragility","level":2,"score":0.46869999170303345},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3714999854564667},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3628999888896942},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3596999943256378},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.34310001134872437},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3280999958515167},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.2944999933242798},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.26420000195503235},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.25679999589920044}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.18066","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18066","pdf_url":"https://arxiv.org/pdf/2608.18066","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.18066","doi":"https://doi.org/10.48550/arxiv.2608.18066","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.18066","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.18066","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.18066","pdf_url":"https://arxiv.org/pdf/2608.18066","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7203757690.pdf","grobid_xml":"https://content.openalex.org/works/W7203757690.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7220938708","https://openalex.org/W7171550432","https://openalex.org/W7220909480","https://openalex.org/W7172400624","https://openalex.org/W7172532696","https://openalex.org/W7160755553","https://openalex.org/W7172518711","https://openalex.org/W7127300183","https://openalex.org/W7223456656","https://openalex.org/W7221062216"],"abstract_inverted_index":{"Memory-based":[0],"self-improving":[1,61,108,226],"agents--those":[2],"that":[3,85,133,158,174,207,250],"learn":[4],"from":[5,257],"an":[6,135],"online":[7],"stream":[8],"of":[9,31,46,53,75,89],"tasks":[10,70],"and":[11,66,101,105,156,160,182,234,248],"improve":[12],"over":[13],"time":[14],"by":[15,171,228],"maintaining":[16],"a":[17,43,107,140],"textual":[18],"memory":[19,155,187],"bank--have":[20],"shown":[21],"great":[22],"promise":[23],"in":[24,98,199,259],"recent":[25],"literature.":[26],"However,":[27],"the":[28,51,69,73,87,118,153,186,196],"reliability":[29],"aspects":[30],"these":[32,79],"methods":[33],"have":[34],"been":[35],"critically":[36],"overlooked.":[37],"In":[38],"this":[39,115,148,165,169,191,213],"work,":[40],"we":[41,81,150],"conduct":[42],"comprehensive":[44],"re-evaluation":[45],"two":[47,56,83],"memory-based":[48],"methods,":[49],"broadening":[50],"scope":[52],"evaluation":[54,94,223],"along":[55],"axes:":[57],"(1)":[58],"including":[59],"multiple":[60,232],"runs":[62,233],"to":[63,71,164,212],"quantify":[64],"variance,":[65],"(2)":[67],"shuffling":[68],"investigate":[72],"effect":[74],"task":[76,125,159],"order.":[77,126],"Through":[78],"experiments,":[80,201],"make":[82],"observations":[84],"expose":[86],"fragility":[88],"current":[90],"methods:":[91],"First,":[92],"agent":[93],"is":[95,121],"inherently":[96],"noisy":[97],"complex":[99],"environments":[100],"on":[102,110,124,243],"multi-step":[103],"tasks,":[104],"stacking":[106],"loop":[109],"top":[111],"can":[112],"further":[113],"amplify":[114],"noise.":[116],"Second,":[117],"agent's":[119],"improvement":[120],"highly":[122],"dependent":[123],"Prior":[127],"works":[128],"often":[129],"adopt":[130],"default":[131],"orderings":[132],"impose":[134],"implicit":[136],"curriculum,":[137],"acting":[138],"as":[139,179],"hidden":[141],"prerequisite":[142],"for":[143,220,225,246],"success.":[144],"To":[145],"better":[146,176],"understand":[147],"fragility,":[149],"manually":[151],"examine":[152],"agents'":[154],"hypothesize":[157],"environment":[161,183],"underspecification":[162,244],"contribute":[163,211],"fragility.":[166,214],"We":[167],"validate":[168],"hypothesis":[170],"incorporating":[172],"information":[173,193],"enables":[175],"specification,":[177],"such":[178],"detailed":[180],"rubrics":[181],"feedback,":[184],"into":[185],"construction":[188],"process.":[189],"While":[190],"added":[192],"partially":[194],"closes":[195],"performance":[197],"degradation":[198],"previous":[200],"significant":[202],"gaps":[203],"still":[204],"remain,":[205],"suggesting":[206],"other":[208],"uncharacterized":[209],"factors":[210],"Looking":[215],"ahead,":[216],"our":[217,241],"work":[218],"advocates":[219],"more":[221],"rigorous":[222],"protocols":[224],"agents":[227,256],"reporting":[229],"results":[230],"across":[231],"stress-testing":[235],"them":[236],"under":[237],"challenging":[238],"conditions.":[239],"Moreover,":[240],"findings":[242],"call":[245],"systems":[247],"interfaces":[249],"enable":[251],"effective":[252],"human":[253],"oversight,":[254],"preventing":[255],"failing":[258],"unforeseeable":[260],"ways.":[261]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-08-20T00:00:00"}