{"id":"https://openalex.org/W4313563681","doi":"https://doi.org/10.1145/3551349.3556943","title":"StandUp4NPR: Standardizing SetUp for Empirically Comparing Neural Program Repair Systems","display_name":"StandUp4NPR: Standardizing SetUp for Empirically Comparing Neural Program Repair Systems","publication_year":2022,"publication_date":"2022-10-10","ids":{"openalex":"https://openalex.org/W4313563681","doi":"https://doi.org/10.1145/3551349.3556943"},"language":"en","primary_location":{"id":"doi:10.1145/3551349.3556943","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551349.3556943","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 37th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Wenkang Zhong","orcid":"https://orcid.org/0000-0003-0145-615X"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wenkang Zhong","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112889244","display_name":"Hongliang Ge","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongliang Ge","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021486962","display_name":"Hongfei Ai","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongfei Ai","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053034433","display_name":"Chuanyi Li","orcid":"https://orcid.org/0000-0003-4956-8142"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chuanyi Li","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374023","display_name":"Kui Liu","orcid":"https://orcid.org/0000-0003-0145-615X"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kui Liu","raw_affiliation_strings":["Huawei Software Engineering Application Technology Lab, China"],"affiliations":[{"raw_affiliation_string":"Huawei Software Engineering Application Technology Lab, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075214885","display_name":"Jidong Ge","orcid":"https://orcid.org/0000-0003-1773-0942"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jidong Ge","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073104000","display_name":"Bin Luo","orcid":"https://orcid.org/0000-0002-9036-0063"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Luo","raw_affiliation_strings":["State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China"],"affiliations":[{"raw_affiliation_string":"State Key Laboratory for Novel Software and Technology, Nanjing University, 22 Hankou Road, Nanjing, China, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I881766915"],"apc_list":null,"apc_paid":null,"fwci":6.0505,"has_fulltext":false,"cited_by_count":23,"citation_normalized_percentile":{"value":0.96772191,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/generalizability-theory","display_name":"Generalizability theory","score":0.8703123331069946},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8356045484542847},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.803189754486084},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6394810676574707},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6102173328399658},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5493990778923035},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.540064811706543},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.5252085328102112},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5153971910476685},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.4753784239292145},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47516199946403503},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10344278812408447}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8703123331069946},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8356045484542847},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.803189754486084},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6394810676574707},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6102173328399658},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5493990778923035},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.540064811706543},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.5252085328102112},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5153971910476685},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.4753784239292145},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47516199946403503},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10344278812408447},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3551349.3556943","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3551349.3556943","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 37th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.4099999964237213}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W2063387237","https://openalex.org/W2122947685","https://openalex.org/W2145373440","https://openalex.org/W2156723666","https://openalex.org/W2272835211","https://openalex.org/W2285839903","https://openalex.org/W2400994325","https://openalex.org/W2465133314","https://openalex.org/W2518136680","https://openalex.org/W2537787699","https://openalex.org/W2560674852","https://openalex.org/W2574841074","https://openalex.org/W2605202003","https://openalex.org/W2736762043","https://openalex.org/W2740349831","https://openalex.org/W2747329762","https://openalex.org/W2762550985","https://openalex.org/W2762844179","https://openalex.org/W2767766265","https://openalex.org/W2769664078","https://openalex.org/W2787871251","https://openalex.org/W2801076109","https://openalex.org/W2850616187","https://openalex.org/W2883977877","https://openalex.org/W2895570420","https://openalex.org/W2904214673","https://openalex.org/W2905489084","https://openalex.org/W2907705732","https://openalex.org/W2924629359","https://openalex.org/W2950571912","https://openalex.org/W2954823997","https://openalex.org/W2963909831","https://openalex.org/W2967591898","https://openalex.org/W2972082064","https://openalex.org/W2973153063","https://openalex.org/W2975950293","https://openalex.org/W2979566992","https://openalex.org/W3039767125","https://openalex.org/W3043761819","https://openalex.org/W3081159607","https://openalex.org/W3084918652","https://openalex.org/W3088927226","https://openalex.org/W3089621332","https://openalex.org/W3098154879","https://openalex.org/W3100698844","https://openalex.org/W3103170042","https://openalex.org/W3104012431","https://openalex.org/W3134686229","https://openalex.org/W3144803734","https://openalex.org/W3161474773","https://openalex.org/W3174220201","https://openalex.org/W3176316046","https://openalex.org/W3193682477","https://openalex.org/W4206070857","https://openalex.org/W4255632703","https://openalex.org/W4288077069","https://openalex.org/W4288614128","https://openalex.org/W4295950999"],"related_works":["https://openalex.org/W2118717649","https://openalex.org/W410723623","https://openalex.org/W2413243053","https://openalex.org/W2015341305","https://openalex.org/W4225593417","https://openalex.org/W2035068594","https://openalex.org/W2573498121","https://openalex.org/W3022298670","https://openalex.org/W3160494304","https://openalex.org/W1549477351"],"abstract_inverted_index":{"Recently,":[0],"the":[1,27,79,118,141,157,181,185,190],"emerging":[2],"trend":[3],"in":[4,188],"automatic":[5],"program":[6],"repair":[7],"is":[8],"to":[9,14,53,75],"apply":[10],"deep":[11],"neural":[12],"networks":[13],"generate":[15],"fixed":[16],"code":[17],"from":[18,221],"buggy":[19],"ones,":[20],"called":[21],"NPR":[22,29,124,132,154,170,193,202],"(Neural":[23],"Program":[24],"Repair).":[25],"However,":[26],"existing":[28,175],"systems":[30,155,171,203],"are":[31,206],"trained":[32],"and":[33,70,91,98,112,115,122,143,160,172],"evaluated":[34],"under":[35],"very":[36],"different":[37,40],"settings":[38],"(e.g.,":[39,204],"training":[41,86,121],"data,":[42,45],"inconsistent":[43],"evaluation":[44,93,113],"wide-ranged":[46],"candidate":[47],"numbers),":[48],"which":[49,178],"makes":[50],"it":[51],"hard":[52],"draw":[54],"fair-enough":[55],"conclusions":[56],"when":[57],"comparing":[58],"them.":[59],"Motivated":[60],"by":[61,134],"this,":[62],"we":[63,145,196,214],"first":[64],"build":[65],"a":[66,85,88,147],"standard":[67],"benchmark":[68,142],"dataset":[69,82],"an":[71,92],"extensive":[72],"framework":[73],"tool":[74,106],"mitigate":[76],"threats":[77],"for":[78],"comparison.":[80],"The":[81,105,162],"consists":[83],"of":[84,102,120,151,168,183,192,201],"set,":[87],"validation":[89],"set":[90,94],"with":[95,210],"144,641,":[96],"13,739":[97],"13,706":[99],"bug-fix":[100],"pairs":[101],"Java":[103],"respectively.":[104],"supports":[107],"selecting":[108],"specific":[109],"training,":[110],"validation,":[111],"datasets":[114],"automatically":[116],"conducting":[117],"pipeline":[119],"evaluating":[123],"models,":[125],"as":[126,128],"well":[127],"easily":[129],"integrating":[130],"new":[131],"models":[133],"implementing":[135],"well-defined":[136],"interfaces.":[137],"Then,":[138],"based":[139],"on":[140],"tool,":[144],"conduct":[146],"comprehensive":[148],"empirical":[149],"comparison":[150],"six":[152],"SOTA":[153],"w.r.t":[156],"repairability,":[158],"inclination":[159],"generalizability.":[161],"experimental":[163,186],"results":[164],"reveal":[165,197],"deeper":[166],"characteristics":[167],"compared":[169],"subvert":[173],"some":[174,198,216],"comparative":[176],"conclusions,":[177],"further":[179],"verify":[180],"necessity":[182],"unifying":[184],"setups":[187],"exploring":[189],"progresses":[191],"systems.":[194],"Meanwhile,":[195],"common":[199],"features":[200],"they":[205],"good":[207],"at":[208],"dealing":[209],"code-delete":[211],"bugs).":[212],"Finally,":[213],"identify":[215],"promising":[217],"research":[218],"directions":[219],"derived":[220],"our":[222],"findings.":[223]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":7}],"updated_date":"2026-04-16T08:26:57.006410","created_date":"2025-10-10T00:00:00"}