{"id":"https://openalex.org/W7154504985","doi":"https://doi.org/10.48550/arxiv.2604.12147","title":"Evaluating Plan Compliance in Autonomous Programming Agents","display_name":"Evaluating Plan Compliance in Autonomous Programming Agents","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7154504985","doi":"https://doi.org/10.48550/arxiv.2604.12147"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.12147","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.12147","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.12147","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5062862631","display_name":"Shuyang Liu","orcid":"https://orcid.org/0000-0002-2678-0979"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shuyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114690119","display_name":"Saman Dehghan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dehghan, Saman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037281393","display_name":"Jatin Ganhotra","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ganhotra, Jatin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079080602","display_name":"Martin Hirzel","orcid":"https://orcid.org/0009-0006-8840-6065"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hirzel, Martin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5058824250","display_name":"Reyhaneh Jabbarvand","orcid":"https://orcid.org/0000-0002-0668-8526"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jabbarvand, Reyhaneh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.4381999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.4381999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.19249999523162842,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11902","display_name":"Intelligent Tutoring Systems and Adaptive Learning","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/plan","display_name":"Plan (archaeology)","score":0.7583000063896179},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5228999853134155},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.5200999975204468},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.34360000491142273},{"id":"https://openalex.org/keywords/compliance","display_name":"Compliance (psychology)","score":0.3418999910354614},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.3337000012397766}],"concepts":[{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.7583000063896179},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5228999853134155},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.5200999975204468},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.5109999775886536},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5030999779701233},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3571000099182129},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.3418999910354614},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3337000012397766},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.32910001277923584},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.3018999993801117},{"id":"https://openalex.org/C48243021","wikidata":"https://www.wikidata.org/wiki/Q932522","display_name":"Strategic planning","level":2,"score":0.3000999987125397},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.289000004529953},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.28760001063346863},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.28679999709129333},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2851000130176544},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.27480000257492065},{"id":"https://openalex.org/C2778321746","wikidata":"https://www.wikidata.org/wiki/Q621922","display_name":"Distrust","level":2,"score":0.27000001072883606},{"id":"https://openalex.org/C167017271","wikidata":"https://www.wikidata.org/wiki/Q7404313","display_name":"Sales and operations planning","level":3,"score":0.2612000107765198},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.25450000166893005}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.12147","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.12147","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.12147","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.12147","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agents":[0],"aspire":[1],"to":[2,19,27,43,68,88,216,231],"eliminate":[3],"the":[4,57,94,144,186,200],"need":[5],"for":[6,24,33],"task-specific":[7,22,223],"prompt":[8],"crafting":[9],"through":[10,75,80],"autonomous":[11],"reason-act-observe":[12],"loops.":[13],"Still,":[14],"they":[15],"are":[16,136],"commonly":[17],"instructed":[18,50,218],"follow":[20,48,217],"a":[21,62,71,89,179,208],"plan":[23,100,121,146,155,159,167,174,180],"guidance,":[25],"e.g.,":[26,83],"resolve":[28],"software":[29],"issues":[30],"following":[31],"phases":[32,184,195],"navigation,":[34],"reproduction,":[35],"patch,":[36],"and":[37,116,150,161,233],"validation.":[38],"Unfortunately,":[39],"it":[40,65],"is":[41,66],"unknown":[42],"what":[44],"extent":[45,58],"agents":[46,59,127],"actually":[47],"such":[49,53],"plans.":[51],"Without":[52,123],"an":[54,124],"analysis,":[55],"determining":[56],"comply":[60],"with":[61,181,199],"given":[63],"plan,":[64,126],"impossible":[67],"assess":[69],"whether":[70],"solution":[72],"was":[73],"reached":[74],"correct":[76],"strategic":[77],"reasoning":[78],"or":[79,86,140],"other":[81],"means,":[82],"data":[84],"contamination":[85],"overfitting":[87],"benchmark.":[90],"This":[91,227],"paper":[92],"presents":[93],"first":[95],"extensive,":[96],"systematic":[97],"analysis":[98],"of":[99],"compliance":[101],"in":[102,185,225],"programming":[103],"agents,":[104],"examining":[105],"16,991":[106],"trajectories":[107],"from":[108],"SWE-agent":[109],"across":[110],"four":[111],"LLMs":[112],"on":[113,130],"SWE-bench":[114,117],"Verified":[115],"Pro":[118],"under":[119],"eight":[120],"variations.":[122],"explicit":[125],"fall":[128],"back":[129],"workflows":[131],"internalized":[132],"during":[133],"training,":[134],"which":[135],"often":[137],"incomplete,":[138],"overfit,":[139],"inconsistently":[141],"applied.":[142],"Providing":[143],"standard":[145],"improves":[147],"issue":[148],"resolution,":[149],"we":[151],"observe":[152],"that":[153,213],"periodic":[154],"reminders":[156],"can":[157,189],"mitigate":[158],"violations":[160],"improve":[162],"task":[163],"success.":[164],"A":[165],"subpar":[166],"hurts":[168],"performance":[169],"even":[170],"more":[171],"than":[172,221,237],"no":[173],"at":[175],"all.":[176],"Surprisingly,":[177],"augmenting":[178],"additional":[182],"task-relevant":[183],"early":[187],"stage":[188],"degrade":[190],"performance,":[191],"particularly":[192],"when":[193],"these":[194],"do":[196],"not":[197],"align":[198],"model's":[201],"internal":[202],"problem-solving":[203],"strategy.":[204],"These":[205],"findings":[206],"highlight":[207],"research":[209],"gap:":[210],"fine-tuning":[211],"paradigms":[212],"teach":[214],"models":[215,230],"plans,":[219],"rather":[220,236],"encoding":[222],"plans":[224],"them.":[226],"requires":[228],"teaching":[229],"reason":[232],"act":[234],"adaptively,":[235],"memorizing":[238],"workflows.":[239]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-16T00:00:00"}