{"id":"https://openalex.org/W4416799210","doi":"https://doi.org/10.1109/snpd65828.2025.11252591","title":"Making the Case for LLM-Generated Automated Program Repair Benchmarks","display_name":"Making the Case for LLM-Generated Automated Program Repair Benchmarks","publication_year":2025,"publication_date":"2025-06-25","ids":{"openalex":"https://openalex.org/W4416799210","doi":"https://doi.org/10.1109/snpd65828.2025.11252591"},"language":null,"primary_location":{"id":"doi:10.1109/snpd65828.2025.11252591","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snpd65828.2025.11252591","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 29th International Conference on Software Engineering, Artificial Intelligence, Networking and Parallel/Distributed Computing (SNPD)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120749483","display_name":"Yasser Ebrahim","orcid":null},"institutions":[{"id":"https://openalex.org/I86519414","display_name":"Algoma University","ror":"https://ror.org/0131d6623","country_code":"CA","type":"education","lineage":["https://openalex.org/I86519414"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Yasser Ebrahim","raw_affiliation_strings":["Algoma University,Computer Science Department,Brampton,Canada"],"affiliations":[{"raw_affiliation_string":"Algoma University,Computer Science Department,Brampton,Canada","institution_ids":["https://openalex.org/I86519414"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5120749483"],"corresponding_institution_ids":["https://openalex.org/I86519414"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.38703627,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"842","last_page":"847"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5220000147819519,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.5220000147819519,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.2759000062942505,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.05900000035762787,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.4957999885082245},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.487199991941452},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.4253000020980835},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4032999873161316},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.39559999108314514},{"id":"https://openalex.org/keywords/quality-assurance","display_name":"Quality assurance","score":0.39410001039505005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6736999750137329},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.4957999885082245},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.487199991941452},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42969998717308044},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.4253000020980835},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4032999873161316},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.39559999108314514},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.39410001039505005},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.375900000333786},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3691999912261963},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.36550000309944153},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.3474999964237213},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.34209999442100525},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29789999127388},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2596000134944916}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/snpd65828.2025.11252591","is_oa":false,"landing_page_url":"https://doi.org/10.1109/snpd65828.2025.11252591","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2025 IEEE/ACIS 29th International Conference on Software Engineering, Artificial Intelligence, Networking and Parallel/Distributed Computing (SNPD)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W2061575154","https://openalex.org/W2246822044","https://openalex.org/W2767431443","https://openalex.org/W2801076109","https://openalex.org/W2953898913","https://openalex.org/W3000617142","https://openalex.org/W4281479826","https://openalex.org/W4288614128","https://openalex.org/W4378942602","https://openalex.org/W4388422146","https://openalex.org/W4401835874","https://openalex.org/W4402457214"],"related_works":[],"abstract_inverted_index":{"Automated":[0],"Program":[1],"Repair":[2],"(APR)":[3],"has":[4],"made":[5],"significant":[6],"strides":[7],"in":[8,85,144],"recent":[9],"years,":[10],"particularly":[11],"with":[12,122],"the":[13,118,129,139],"integration":[14],"of":[15,41,120],"large":[16],"language":[17],"models":[18],"(LLMs)":[19],"and":[20,62,109,114,135],"deep":[21],"learning":[22],"techniques.":[23],"Yet":[24],"despite":[25],"this":[26,70,126],"progress,":[27],"one":[28],"fundamental":[29],"issue":[30],"continues":[31],"to":[32],"hinder":[33],"advancement:":[34],"how":[35,92],"we":[36,72,101],"evaluate":[37],"these":[38,78,98],"systems.":[39],"Many":[40],"today\u2019s":[42],"APR":[43,145],"benchmarks":[44,94,108],"suffer":[45],"from":[46],"serious":[47],"limitations\u2014including":[48],"small":[49],"dataset":[50],"sizes,":[51],"synthetic":[52],"or":[53],"unrealistic":[54],"bug":[55],"scenarios,":[56],"overfitting":[57],"risks,":[58],"ambiguous":[59],"evaluation":[60,137],"criteria,":[61],"a":[63,74,111],"narrow":[64],"focus":[65],"on":[66],"certain":[67],"programming":[68],"languages.In":[69],"paper,":[71],"take":[73],"critical":[75],"look":[76],"at":[77],"challenges":[79],"by":[80],"identifying":[81],"eight":[82],"core":[83],"limitations":[84],"widely":[86],"used":[87],"benchmarks.":[88],"We":[89],"then":[90],"explore":[91],"LLM-generated":[93,107],"can":[95],"help":[96],"overcome":[97],"obstacles.":[99],"Finally,":[100],"address":[102],"some":[103],"potential":[104],"concerns":[105],"about":[106],"propose":[110],"quality":[112],"assurance":[113],"validation":[115],"framework.By":[116],"combining":[117],"strengths":[119],"LLMs":[121],"thoughtful":[123],"benchmark":[124],"design,":[125],"work":[127],"lays":[128],"foundation":[130],"for":[131,141],"more":[132],"robust,":[133],"diverse,":[134],"meaningful":[136],"frameworks\u2014paving":[138],"way":[140],"future":[142],"breakthroughs":[143],"research.":[146]},"counts_by_year":[],"updated_date":"2026-03-07T16:01:11.037858","created_date":"2025-11-28T00:00:00"}