{"id":"https://openalex.org/W7169516707","doi":"https://doi.org/10.1145/3803437.3805536","title":"Bugs in Modern LLM Agent Frameworks: An Empirical Study","display_name":"Bugs in Modern LLM Agent Frameworks: An Empirical Study","publication_year":2026,"publication_date":"2026-07-05","ids":{"openalex":"https://openalex.org/W7169516707","doi":"https://doi.org/10.1145/3803437.3805536"},"language":null,"primary_location":{"id":"doi:10.1145/3803437.3805536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3803437.3805536","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141077605","display_name":"Xinxue Zhu","orcid":"https://orcid.org/0009-0001-4945-2820"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xinxue Zhu","raw_affiliation_strings":["Nantong University, Nantong, China"],"raw_orcid":"https://orcid.org/0009-0001-4945-2820","affiliations":[{"raw_affiliation_string":"Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103089404","display_name":"Jiacong Wu","orcid":"https://orcid.org/0009-0009-8114-6610"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiacong Wu","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0006-5160-6418","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092424261","display_name":"Xiaoyu Zhang","orcid":"https://orcid.org/0000-0001-7010-6749"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xiaoyu Zhang","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7010-6749","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101674538","display_name":"Tianlin Li","orcid":"https://orcid.org/0000-0002-2207-1622"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianlin Li","raw_affiliation_strings":["Beihang University, BeiJing, China"],"raw_orcid":"https://orcid.org/0000-0002-2207-1622","affiliations":[{"raw_affiliation_string":"Beihang University, BeiJing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075895554","display_name":"Yanzhou Mu","orcid":"https://orcid.org/0000-0003-1816-2246"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanzhou Mu","raw_affiliation_strings":["Nanjing university, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-1816-2246","affiliations":[{"raw_affiliation_string":"Nanjing university, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071575216","display_name":"Juan Zhai","orcid":"https://orcid.org/0000-0001-5017-8016"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Juan Zhai","raw_affiliation_strings":["University of Massachusetts Amherst, Amherst, USA"],"raw_orcid":"https://orcid.org/0000-0001-5017-8016","affiliations":[{"raw_affiliation_string":"University of Massachusetts Amherst, Amherst, USA","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101843177","display_name":"Chao Shen","orcid":"https://orcid.org/0000-0002-6959-0569"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Shen","raw_affiliation_strings":["Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-6959-0569","affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075174750","display_name":"Chunrong Fang","orcid":"https://orcid.org/0000-0002-9930-7111"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chunrong Fang","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-9930-7111","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100355692","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7300-9215","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1588","last_page":"1592"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":null,"topics":[],"keywords":[{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.650600016117096},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.550000011920929},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5027999877929688},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.3652999997138977},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.364300012588501},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.3547999858856201},{"id":"https://openalex.org/keywords/empirical-evidence","display_name":"Empirical evidence","score":0.3375999927520752}],"concepts":[{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.650600016117096},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.550000011920929},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5343999862670898},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5027999877929688},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.42570000886917114},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3725000023841858},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3695000112056732},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3547999858856201},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.35199999809265137},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.3375999927520752},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.32989999651908875},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3077999949455261},{"id":"https://openalex.org/C2780873155","wikidata":"https://www.wikidata.org/wiki/Q392811","display_name":"Agent-based model","level":2,"score":0.3050000071525574},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.30469998717308044},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.27959999442100525},{"id":"https://openalex.org/C130963320","wikidata":"https://www.wikidata.org/wiki/Q1401207","display_name":"Root cause analysis","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C84945661","wikidata":"https://www.wikidata.org/wiki/Q7366567","display_name":"Root cause","level":2,"score":0.26350000500679016},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2621999979019165}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3803437.3805536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3803437.3805536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3803437.3805536","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th ACM International Conference on the Foundations of Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W4405366282","https://openalex.org/W7125983739"],"related_works":[],"abstract_inverted_index":{"LLM":[0],"agents":[1],"have":[2],"been":[3],"widely":[4],"adopted":[5],"in":[6,25,104],"real-world":[7],"applications,":[8],"relying":[9],"on":[10,37],"agent":[11,28,74,89],"frameworks":[12,29],"for":[13],"lifecycle":[14,75],"execution":[15],"and":[16,58,60,68,83,99,115,123],"multiagent":[17],"coordination.":[18],"As":[19],"these":[20],"systems":[21],"scale,":[22],"understanding":[23],"bugs":[24,91],"the":[26,105],"underlying":[27],"becomes":[30],"critical.":[31],"However,":[32],"existing":[33],"work":[34],"mainly":[35,92],"focuses":[36],"agent-level":[38],"failures,":[39],"overlooking":[40],"framework-level":[41],"bugs.":[42],"To":[43],"address":[44],"this":[45],"gap,":[46],"we":[47],"conduct":[48],"an":[49],"empirical":[50],"study":[51],"of":[52,64],"998":[53],"issue":[54],"reports":[55],"from":[56,94],"CrewAI":[57],"LangGraph,":[59],"construct":[61],"a":[62],"taxonomy":[63],"15":[65],"root":[66],"causes":[67],"seven":[69],"observable":[70],"symptoms":[71],"across":[72],"five":[73],"stages:":[76],"'Agent":[77],"Initialization',":[78],"'Perception',":[79],"'Self-Action',":[80],"'Mutual":[81],"Interaction'":[82],"'Evolution'.":[84],"Our":[85],"findings":[86],"show":[87],"that":[88],"framework":[90],"arise":[93],"'API":[95,97],"Misuse',":[96],"Incompatibility',":[98],"'Documentation":[100],"Desync',":[101],"largely":[102],"concentrated":[103],"'Self-Action'":[106],"stage.":[107],"Symptoms":[108],"typically":[109],"appear":[110],"as":[111],"'Functional":[112],"Error',":[113],"'Crash',":[114],"'Build":[116],"Failure',":[117],"reflecting":[118],"disruptions":[119],"to":[120],"task":[121],"progression":[122],"control":[124],"flow.":[125]},"counts_by_year":[],"updated_date":"2026-07-18T05:58:53.570196","created_date":"2026-07-18T00:00:00"}