{"id":"https://openalex.org/W4407749710","doi":"https://doi.org/10.1145/3704997","title":"Automated Program Repair: Emerging Trends Pose and Expose Problems for Benchmarks","display_name":"Automated Program Repair: Emerging Trends Pose and Expose Problems for Benchmarks","publication_year":2025,"publication_date":"2025-02-19","ids":{"openalex":"https://openalex.org/W4407749710","doi":"https://doi.org/10.1145/3704997"},"language":"en","primary_location":{"id":"doi:10.1145/3704997","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3704997","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057833944","display_name":"Joseph Renzullo","orcid":"https://orcid.org/0009-0002-2534-8700"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joseph Renzullo","raw_affiliation_strings":["School of Computing and Augmented Intelligence, Arizona State University, Tempe, United States"],"raw_orcid":"https://orcid.org/0009-0002-2534-8700","affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence, Arizona State University, Tempe, United States","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066333300","display_name":"Pemma Reiter","orcid":"https://orcid.org/0000-0003-1425-7811"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pemma Reiter","raw_affiliation_strings":["Arizona State University, Tempe, United States"],"raw_orcid":"https://orcid.org/0000-0003-1425-7811","affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, United States","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083601969","display_name":"Westley Weimer","orcid":"https://orcid.org/0000-0001-6749-2204"},"institutions":[{"id":"https://openalex.org/I27837315","display_name":"University of Michigan","ror":"https://ror.org/00jmfr291","country_code":"US","type":"education","lineage":["https://openalex.org/I27837315"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Westley Weimer","raw_affiliation_strings":["University of Michigan, Ann Arbor, United States"],"raw_orcid":"https://orcid.org/0000-0001-6749-2204","affiliations":[{"raw_affiliation_string":"University of Michigan, Ann Arbor, United States","institution_ids":["https://openalex.org/I27837315"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114376382","display_name":"Stephanie Forrest","orcid":"https://orcid.org/0000-0002-5904-1646"},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephanie Forrest","raw_affiliation_strings":["Arizona State University, Tempe, United States"],"raw_orcid":"https://orcid.org/0000-0002-5904-1646","affiliations":[{"raw_affiliation_string":"Arizona State University, Tempe, United States","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.8466,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.9579765,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"57","issue":"8","first_page":"1","last_page":"18"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11005","display_name":"Radiation Effects in Electronics","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8839205503463745},{"id":"https://openalex.org/keywords/expose","display_name":"EXPOSE","score":0.8694818019866943},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33112967014312744}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8839205503463745},{"id":"https://openalex.org/C2779137570","wikidata":"https://www.wikidata.org/wiki/Q16243196","display_name":"EXPOSE","level":2,"score":0.8694818019866943},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33112967014312744},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3704997","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3704997","pdf_url":null,"source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W1902237438","https://openalex.org/W2041190309","https://openalex.org/W2131774270","https://openalex.org/W2145373440","https://openalex.org/W2156723666","https://openalex.org/W2298313545","https://openalex.org/W2526529994","https://openalex.org/W2736287575","https://openalex.org/W2766078762","https://openalex.org/W2794443436","https://openalex.org/W2795030435","https://openalex.org/W2801865415","https://openalex.org/W2883977877","https://openalex.org/W2886081451","https://openalex.org/W2911270308","https://openalex.org/W2912218663","https://openalex.org/W2912296587","https://openalex.org/W2920807444","https://openalex.org/W2958228760","https://openalex.org/W2961757301","https://openalex.org/W2962818281","https://openalex.org/W2963909831","https://openalex.org/W2964210007","https://openalex.org/W2972704946","https://openalex.org/W2975950293","https://openalex.org/W2981875947","https://openalex.org/W2998011150","https://openalex.org/W3003756849","https://openalex.org/W3032749142","https://openalex.org/W3035290244","https://openalex.org/W3086449553","https://openalex.org/W3089853590","https://openalex.org/W3094130708","https://openalex.org/W3100191342","https://openalex.org/W3112198431","https://openalex.org/W3115042282","https://openalex.org/W3137004499","https://openalex.org/W3156480510","https://openalex.org/W3160155705","https://openalex.org/W3177722508","https://openalex.org/W3181414820","https://openalex.org/W3206322347","https://openalex.org/W3208407575","https://openalex.org/W3208624098","https://openalex.org/W4213423286","https://openalex.org/W4226104947","https://openalex.org/W4229046772","https://openalex.org/W4239072543","https://openalex.org/W4281763794","https://openalex.org/W4283796010","https://openalex.org/W4284666365","https://openalex.org/W4284674057","https://openalex.org/W4300771635","https://openalex.org/W4300960622","https://openalex.org/W4312490312","https://openalex.org/W4313563520","https://openalex.org/W4313563681","https://openalex.org/W4323896452","https://openalex.org/W4366371549","https://openalex.org/W4391558518"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2356093757","https://openalex.org/W2059618490","https://openalex.org/W3005909949","https://openalex.org/W3182926544","https://openalex.org/W2791072330","https://openalex.org/W2073173259","https://openalex.org/W2314887962"],"abstract_inverted_index":{"Machine":[0],"learning":[1],"(ML)":[2],"pervades":[3],"the":[4,43,59,102,113,116,153,195],"field":[5,196],"of":[6,36,45,119,136,155],"Automated":[7],"Program":[8],"Repair":[9],"(APR).":[10],"Algorithms":[11],"deploy":[12],"neural":[13],"machine":[14],"translation":[15],"and":[16,38,51,81,130,133,148,176,179],"large":[17,80],"language":[18],"models":[19],"(LLMs)":[20],"to":[21,53,182],"generate":[22],"software":[23],"patches,":[24],"among":[25],"other":[26],"tasks.":[27],"But,":[28],"there":[29],"are":[30,49,92,126,138,186],"important":[31],"differences":[32],"between":[33],"these":[34,141,166],"applications":[35],"ML":[37,69,120],"earlier":[39],"work,":[40],"which":[41,90,162],"complicates":[42],"task":[44],"ensuring":[46],"that":[47,58,140,194],"results":[48],"valid":[50],"likely":[52],"generalize.":[54],"A":[55],"challenge":[56,154],"is":[57,74,197],"most":[60],"popular":[61],"APR":[62,99,158],"evaluation":[63,174],"benchmarks":[64],"were":[65,163,168],"not":[66],"designed":[67,164],"with":[68],"techniques":[70,167],"in":[71,98,101,112,173],"mind.":[72],"This":[73,94],"especially":[75],"true":[76],"for":[77],"LLMs,":[78],"whose":[79],"often":[82],"poorly-disclosed":[83],"training":[84],"datasets":[85],"may":[86],"include":[87],"problems":[88],"on":[89],"they":[91,185],"evaluated.":[93],"article":[95],"reviews":[96],"work":[97],"published":[100],"field\u2019s":[103],"top":[104],"five":[105],"venues":[106],"since":[107],"2018,":[108],"emphasizing":[109],"emerging":[110],"trends":[111],"field,":[114],"including":[115,122],"dramatic":[117],"rise":[118],"models,":[121],"LLMs.":[123],"ML-based":[124,157],"articles":[125],"categorized":[127],"along":[128],"structural":[129],"functional":[131],"dimensions,":[132],"a":[134],"variety":[135],"issues":[137],"identified":[139],"new":[142,192],"methods":[143],"raise.":[144],"Importantly,":[145],"data":[146],"leakage":[147],"contamination":[149],"concerns":[150],"arise":[151],"from":[152],"validating":[156],"using":[159],"existing":[160],"benchmarks,":[161],"before":[165],"popular.":[169],"We":[170],"discuss":[171],"inconsistencies":[172],"design":[175],"performance":[177],"reporting":[178],"offer":[180],"pointers":[181],"solutions":[183],"where":[184],"available.":[187],"Finally,":[188],"we":[189],"highlight":[190],"promising":[191],"directions":[193],"already":[198],"taking.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}