{"id":"https://openalex.org/W4389208841","doi":"https://doi.org/10.1145/3611643.3616331","title":"A Large-Scale Empirical Review of Patch Correctness Checking Approaches","display_name":"A Large-Scale Empirical Review of Patch Correctness Checking Approaches","publication_year":2023,"publication_date":"2023-11-30","ids":{"openalex":"https://openalex.org/W4389208841","doi":"https://doi.org/10.1145/3611643.3616331"},"language":"en","primary_location":{"id":"doi:10.1145/3611643.3616331","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616331","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107247519","display_name":"Jun Yang","orcid":"https://orcid.org/0009-0006-3657-6169"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jun Yang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101778578","display_name":"Yuehan Wang","orcid":"https://orcid.org/0009-0004-0988-9835"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuehan Wang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024354460","display_name":"Yiling Lou","orcid":"https://orcid.org/0000-0002-4066-3365"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiling Lou","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003031253","display_name":"Ming Wen","orcid":"https://orcid.org/0000-0001-5588-9618"},"institutions":[{"id":"https://openalex.org/I47720641","display_name":"Huazhong University of Science and Technology","ror":"https://ror.org/00p991c53","country_code":"CN","type":"education","lineage":["https://openalex.org/I47720641"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ming Wen","raw_affiliation_strings":["Huazhong University of Science and Technology, Wuhan, China"],"affiliations":[{"raw_affiliation_string":"Huazhong University of Science and Technology, Wuhan, China","institution_ids":["https://openalex.org/I47720641"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043546718","display_name":"Lingming Zhang","orcid":"https://orcid.org/0000-0001-5175-2702"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lingming Zhang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5107247519"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":1.9426,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.91368479,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1203","last_page":"1215"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11005","display_name":"Radiation Effects in Electronics","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8562390804290771},{"id":"https://openalex.org/keywords/bytecode","display_name":"Bytecode","score":0.8315033316612244},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.7937825918197632},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.7205101847648621},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4962223172187805},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45700040459632874},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.4426148533821106},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4384012520313263},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.42341727018356323},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.41692715883255005},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.35791724920272827},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.18176066875457764}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8562390804290771},{"id":"https://openalex.org/C2779818221","wikidata":"https://www.wikidata.org/wiki/Q837330","display_name":"Bytecode","level":3,"score":0.8315033316612244},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.7937825918197632},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.7205101847648621},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4962223172187805},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45700040459632874},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.4426148533821106},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4384012520313263},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.42341727018356323},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.41692715883255005},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35791724920272827},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.18176066875457764},{"id":"https://openalex.org/C548217200","wikidata":"https://www.wikidata.org/wiki/Q251","display_name":"Java","level":2,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3611643.3616331","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3611643.3616331","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9","score":0.6200000047683716}],"awards":[{"id":"https://openalex.org/G1100817172","display_name":null,"funder_award_id":"619320","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1121271761","display_name":null,"funder_award_id":"Program","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1231421488","display_name":null,"funder_award_id":"under","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1888249465","display_name":null,"funder_award_id":"61932021","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2087396116","display_name":null,"funder_award_id":"China","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2779298988","display_name":null,"funder_award_id":"No.62002125 and No.61932021","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2802911279","display_name":null,"funder_award_id":"Young","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3317480652","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G351720009","display_name":null,"funder_award_id":"2002125","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4291562662","display_name":null,"funder_award_id":"CCF-2131943 and CCF-2141474","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5994120800","display_name":null,"funder_award_id":"Natural","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6163997753","display_name":null,"funder_award_id":"62002125","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6665774762","display_name":null,"funder_award_id":"2021QNRC001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8714109953","display_name":null,"funder_award_id":"QNRC001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8863666567","display_name":null,"funder_award_id":"and No.","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320332222","display_name":"University of Illinois at Urbana-Champaign","ror":"https://ror.org/047426m28"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W1912569824","https://openalex.org/W1950030762","https://openalex.org/W1963523113","https://openalex.org/W1965194038","https://openalex.org/W1966716734","https://openalex.org/W1971650562","https://openalex.org/W1976526581","https://openalex.org/W1977696633","https://openalex.org/W2054751837","https://openalex.org/W2060333670","https://openalex.org/W2061575154","https://openalex.org/W2063387237","https://openalex.org/W2132083787","https://openalex.org/W2142403498","https://openalex.org/W2145373440","https://openalex.org/W2153881107","https://openalex.org/W2156723666","https://openalex.org/W2285839903","https://openalex.org/W2344973853","https://openalex.org/W2373227884","https://openalex.org/W2400994325","https://openalex.org/W2465133314","https://openalex.org/W2518136680","https://openalex.org/W2548997977","https://openalex.org/W2735571786","https://openalex.org/W2740646481","https://openalex.org/W2741328617","https://openalex.org/W2767431443","https://openalex.org/W2767766265","https://openalex.org/W2794443436","https://openalex.org/W2795030435","https://openalex.org/W2795338679","https://openalex.org/W2795866244","https://openalex.org/W2804618777","https://openalex.org/W2850616187","https://openalex.org/W2851896161","https://openalex.org/W2867448323","https://openalex.org/W2886412171","https://openalex.org/W2895570420","https://openalex.org/W2904214673","https://openalex.org/W2905489084","https://openalex.org/W2907705732","https://openalex.org/W2924629359","https://openalex.org/W2952920225","https://openalex.org/W2953998535","https://openalex.org/W2954778170","https://openalex.org/W2955555786","https://openalex.org/W2963909831","https://openalex.org/W2967591898","https://openalex.org/W2975950293","https://openalex.org/W2991748713","https://openalex.org/W3011013751","https://openalex.org/W3012438487","https://openalex.org/W3042956498","https://openalex.org/W3043761819","https://openalex.org/W3089621332","https://openalex.org/W3100698844","https://openalex.org/W3104012431","https://openalex.org/W3105867435","https://openalex.org/W3121734253","https://openalex.org/W3122473522","https://openalex.org/W3125046082","https://openalex.org/W3161474773","https://openalex.org/W3182136221","https://openalex.org/W3193682477","https://openalex.org/W4280528532","https://openalex.org/W4284674057","https://openalex.org/W4288080276","https://openalex.org/W4308643319","https://openalex.org/W4312537169","https://openalex.org/W4382498938","https://openalex.org/W4384345708","https://openalex.org/W6894264461"],"related_works":["https://openalex.org/W1574414179","https://openalex.org/W4362597605","https://openalex.org/W3009056573","https://openalex.org/W2922073769","https://openalex.org/W4297676672","https://openalex.org/W4281702477","https://openalex.org/W4378510483","https://openalex.org/W4376166922","https://openalex.org/W2490526372","https://openalex.org/W4221142204"],"abstract_inverted_index":{"Automated":[0],"Program":[1],"Repair":[2],"(APR)":[3],"techniques":[4,52,179,195,215,227,252,258],"have":[5,53],"drawn":[6],"wide":[7],"attention":[8],"from":[9,138,168,219],"both":[10],"academia":[11],"and":[12,150,246,259],"industry.":[13],"Meanwhile,":[14],"one":[15],"main":[16],"limitation":[17],"with":[18,242],"the":[19,29,35,42,71,86,139,188,220,248,280],"current":[20],"state-of-the-art":[21,125,213],"APR":[22,72,142,171],"tools":[23,73],"is":[24],"that":[25,196],"patches":[26,78,136,157,198,208,241],"passing":[27],"all":[28,123,154],"original":[30],"tests":[31],"are":[32,63,199],"not":[33],"necessarily":[34],"correct":[36,197],"ones":[37],"wanted":[38],"by":[39,191],"developers,":[40],"i.e.,":[41],"plausible":[43,156,207],"patch":[44,148],"problem.":[45],"To":[46],"date,":[47],"various":[48,169,184,274],"Patch-Correctness":[49],"Checking":[50],"(PCC)":[51],"been":[54],"proposed":[55],"to":[56,95,98,118,121,202,217],"address":[57],"this":[58,103],"important":[59],"issue.":[60],"However,":[61],"they":[62],"only":[64,80],"evaluated":[65],"on":[66,180,232,240,268],"very":[67,249],"limited":[68],"datasets":[69],"as":[70],"used":[74],"for":[75,265,276],"generating":[76],"such":[77],"can":[79,151,253],"explore":[81,153],"a":[82,262],"small":[83],"subset":[84],"of":[85,89,176],"search":[87,162],"space":[88,163],"possible":[90,155],"patches,":[91],"posing":[92],"serious":[93],"threats":[94],"external":[96],"validity":[97],"existing":[99,192],"PCC":[100,109,116,126,131,178,194,278],"studies.":[101],"In":[102],"paper,":[104],"we":[105,271],"construct":[106],"an":[107],"extensive":[108,174],"dataset,":[110,235],"PraPatch":[111,133,181],"(the":[112],"largest":[113],"manually":[114],"labeled":[115],"dataset":[117,132,221],"our":[119,130,233,269],"knowledge),":[120],"revisit":[122],"nine":[124],"techniques.":[127],"More":[128],"specifically,":[129],"includes":[134],"1,988":[135],"generated":[137],"recent":[140,250],"PraPR":[141],"tool,":[143],"which":[144],"leverages":[145],"highly-optimized":[146],"bytecode-level":[147],"executions":[149],"exhaustively":[152],"within":[158],"its":[159],"large":[160],"predefined":[161],"(including":[164],"well-known":[165],"fixing":[166],"patterns":[167],"prior":[170],"tools).":[172],"Our":[173],"study":[175],"representative":[177],"has":[182],"revealed":[183],"findings,":[185,270],"including:":[186],"1)":[187],"assumption":[189],"made":[190],"static":[193,257],"more":[200,243],"similar":[201],"buggy":[203],"code":[204],"than":[205],"incorrect":[206],"no":[209],"longer":[210],"holds,":[211],"2)":[212],"learning-based":[214],"tend":[216],"suffer":[218],"overfitting":[222],"problem,":[223],"3)":[224],"while":[225],"dynamic":[226],"overall":[228],"retain":[229],"their":[230,236],"effectiveness":[231],"new":[234],"performance":[237],"drops":[238],"substantially":[239,254],"complicated":[244],"changes":[245],"4)":[247],"naturalness-based":[251],"outperform":[255],"traditional":[256],"could":[260],"be":[261],"promising":[263],"direction":[264],"PCC.":[266],"Based":[267],"also":[272],"provide":[273],"guidelines/suggestions":[275],"advancing":[277],"in":[279],"near":[281],"future.":[282]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":4}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}