{"id":"https://openalex.org/W7166031596","doi":"https://doi.org/10.48550/arxiv.2606.26978","title":"To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair","display_name":"To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair","publication_year":2026,"publication_date":"2026-06-25","ids":{"openalex":"https://openalex.org/W7166031596","doi":"https://doi.org/10.48550/arxiv.2606.26978"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.26978","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26978","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.26978","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132757273","display_name":"Zhihao Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Zhihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100939152","display_name":"Junhua Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Junhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139392617","display_name":"Mingyi Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Mingyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139407125","display_name":"Xin Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078038118","display_name":"Zhensu Sun","orcid":"https://orcid.org/0000-0001-5393-7858"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Zhensu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050796169","display_name":"Renyu Yang","orcid":"https://orcid.org/0000-0001-6334-4925"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Renyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139455446","display_name":"David Lo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lo, David","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139463027","display_name":"Li Erran Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8531000018119812,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.8531000018119812,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.0560000017285347,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.01510000042617321,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5593000054359436},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4661000072956085},{"id":"https://openalex.org/keywords/execution-time","display_name":"Execution time","score":0.4422000050544739},{"id":"https://openalex.org/keywords/execution-model","display_name":"Execution model","score":0.37709999084472656},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.3702000081539154},{"id":"https://openalex.org/keywords/resource","display_name":"Resource (disambiguation)","score":0.3571999967098236}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8222000002861023},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5593000054359436},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4661000072956085},{"id":"https://openalex.org/C2989134064","wikidata":"https://www.wikidata.org/wiki/Q288510","display_name":"Execution time","level":2,"score":0.4422000050544739},{"id":"https://openalex.org/C2776834041","wikidata":"https://www.wikidata.org/wiki/Q25346349","display_name":"Execution model","level":2,"score":0.37709999084472656},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3702000081539154},{"id":"https://openalex.org/C206345919","wikidata":"https://www.wikidata.org/wiki/Q20380951","display_name":"Resource (disambiguation)","level":2,"score":0.3571999967098236},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.30250000953674316},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2896000146865845},{"id":"https://openalex.org/C200130814","wikidata":"https://www.wikidata.org/wiki/Q362858","display_name":"Worst-case execution time","level":3,"score":0.288100004196167},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.28519999980926514},{"id":"https://openalex.org/C98183937","wikidata":"https://www.wikidata.org/wiki/Q2112188","display_name":"Program analysis","level":2,"score":0.25940001010894775},{"id":"https://openalex.org/C29202148","wikidata":"https://www.wikidata.org/wiki/Q287260","display_name":"Resource allocation","level":2,"score":0.25859999656677246}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.26978","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26978","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.26978","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.26978","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6776006817817688}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"LLM-based":[0,58],"agents":[1,42,90,125,143,177,218],"for":[2,104],"program":[3,59],"repair":[4,82,173],"are":[5],"increasingly":[6],"built":[7],"on":[8,40,172,175,225],"a":[9,50,105,238,246],"\"generate-run-revise\"":[10],"paradigm,":[11],"iteratively":[12],"executing":[13],"tests":[14],"to":[15,151],"evaluate":[16,79],"and":[17,35,88,94,110,126,144,155,186,193,202],"refine":[18],"patches.":[19],"This":[20],"execution-based":[21],"approach":[22],"has":[23],"become":[24],"standard":[25],"practice":[26],"in":[27,57],"state-of-the-art":[28],"systems.":[29],"However,":[30],"executions":[31,157],"can":[32],"be":[33,235],"time-consuming":[34],"expensive,":[36],"yet":[37],"their":[38],"impact":[39],"these":[41],"remains":[43],"underexplored.":[44],"In":[45],"this":[46],"paper,":[47],"we":[48,67,78],"conduct":[49],"two-stage":[51],"empirical":[52],"study":[53],"over":[54],"execution":[55,63,100,120,220],"behavior":[56,64,139],"repair.":[60],"To":[61],"characterize":[62],"at":[65],"scale,":[66],"first":[68],"analyze":[69],"7,745":[70],"agent":[71],"traces":[72],"from":[73,149],"SWE-bench":[74,86],"leaderboard":[75],"submissions.":[76],"Second,":[77],"3,000":[80],"end-to-end":[81],"attempts":[83],"across":[84,123,142],"200":[85],"instances":[87,226],"three":[89,115],"(Claude":[91],"Code,":[92],"Codex,":[93],"the":[95,181],"open-source":[96],"OpenCode)":[97],"under":[98],"four":[99],"paradigms,":[101],"which":[102],"allows":[103],"fine-grained":[106],"comparison":[107],"of":[108,132],"performance":[109],"cost.":[111,204],"Our":[112],"analysis":[113],"reveals":[114],"key":[116],"observations:":[117],"(1)":[118],"Code":[119],"is":[121,188,208],"used":[122],"all":[124],"models":[127,180],"analyzed,":[128],"with":[129,146,178,240],"an":[130,241],"average":[131],"8.8":[133],"test":[134],"runs":[135],"per":[136,153],"task.":[137],"Execution":[138,167,206],"varies":[140],"substantially":[141],"models,":[145],"frequency":[147],"ranging":[148],"2":[150],"19":[152],"task,":[154],"late-stage":[156],"consistently":[158],"achieve":[159],"higher":[160],"success":[161],"rates":[162],"than":[163,211],"early-stage":[164],"ones.":[165],"(2)":[166],"restrictions":[168],"have":[169],"little":[170,230],"effect":[171],"success:":[174],"commercial":[176],"SOTA":[179],"resolve-rate":[182],"gap":[183],"between":[184],"Prohibited":[185,198],"Unrestricted":[187],"only":[189],"1.25":[190],"percentage":[191],"points":[192],"not":[194,245],"statistically":[195],"significant,":[196],"while":[197],"saves":[199],"substantial":[200],"token":[201],"wall-clock":[203],"(3)":[205],"benefit":[207],"concentrated":[209],"rather":[210],"uniform.":[212],"These":[213],"patterns":[214],"suggest":[215],"that":[216],"current":[217],"apply":[219],"indiscriminately,":[221],"paying":[222],"its":[223],"cost":[224],"where":[227],"it":[228],"provides":[229],"benefit.":[231],"Execution,":[232],"therefore,":[233],"should":[234],"treated":[236],"as":[237],"resource":[239],"explicit":[242],"cost-benefit":[243],"tradeoff,":[244],"default":[247],"capability.":[248]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-27T00:00:00"}