{"id":"https://openalex.org/W3032749142","doi":"https://doi.org/10.1109/tse.2020.2998785","title":"Quality of Automated Program Repair on Real-World Defects","display_name":"Quality of Automated Program Repair on Real-World Defects","publication_year":2020,"publication_date":"2020-06-01","ids":{"openalex":"https://openalex.org/W3032749142","doi":"https://doi.org/10.1109/tse.2020.2998785","mag":"3032749142"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2020.2998785","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2020.2998785","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103152221","display_name":"Manish Motwani","orcid":"https://orcid.org/0000-0001-5129-3980"},"institutions":[{"id":"https://openalex.org/I177605424","display_name":"Amherst College","ror":"https://ror.org/028vqfs63","country_code":"US","type":"education","lineage":["https://openalex.org/I177605424"]},{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Manish Motwani","raw_affiliation_strings":["Manning College of Information and Computer Sciences, University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5129-3980","affiliations":[{"raw_affiliation_string":"Manning College of Information and Computer Sciences, University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I177605424","https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103098504","display_name":"Mauricio Soto","orcid":"https://orcid.org/0000-0002-1196-1646"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mauricio Soto","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","[School of Computer Science, Carnegie Mellon University, 6612 Pittsburgh, Pennsylvania United States (e-mail: msotogon@cs.cmu.edu)]"],"raw_orcid":"https://orcid.org/0000-0002-1196-1646","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"[School of Computer Science, Carnegie Mellon University, 6612 Pittsburgh, Pennsylvania United States (e-mail: msotogon@cs.cmu.edu)]","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028789964","display_name":"Yuriy Brun","orcid":"https://orcid.org/0000-0003-3027-7986"},"institutions":[{"id":"https://openalex.org/I177605424","display_name":"Amherst College","ror":"https://ror.org/028vqfs63","country_code":"US","type":"education","lineage":["https://openalex.org/I177605424"]},{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuriy Brun","raw_affiliation_strings":["Manning College of Information and Computer Sciences, University of Massachusetts Amherst, Amherst, MA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3027-7986","affiliations":[{"raw_affiliation_string":"Manning College of Information and Computer Sciences, University of Massachusetts Amherst, Amherst, MA, USA","institution_ids":["https://openalex.org/I177605424","https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088079823","display_name":"Ren\u00e9 Just","orcid":"https://orcid.org/0000-0002-5982-275X"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rene Just","raw_affiliation_strings":["Paul G. Allen School of Computer Science and Engineering, University of Washington, Seattle, WA, USA","[Paul G. Allen School of Computer Science & Engineering, University of Washington, 7284 Seattle, Washington United States (e-mail: rjust@cs.washington.edu)]"],"raw_orcid":"https://orcid.org/0000-0002-5982-275X","affiliations":[{"raw_affiliation_string":"Paul G. Allen School of Computer Science and Engineering, University of Washington, Seattle, WA, USA","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"[Paul G. Allen School of Computer Science & Engineering, University of Washington, 7284 Seattle, Washington United States (e-mail: rjust@cs.washington.edu)]","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032356672","display_name":"Claire Le Goues","orcid":"https://orcid.org/0000-0002-3931-060X"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Claire Le Goues","raw_affiliation_strings":["School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","[School of Computer Science, Carnegie Mellon University, 6612 Pittsburgh, Pennsylvania United States (e-mail: clegoues@cs.cmu.edu)]"],"raw_orcid":"https://orcid.org/0000-0002-3931-060X","affiliations":[{"raw_affiliation_string":"School of Computer Science, Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"[School of Computer Science, Carnegie Mellon University, 6612 Pittsburgh, Pennsylvania United States (e-mail: clegoues@cs.cmu.edu)]","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5103152221"],"corresponding_institution_ids":["https://openalex.org/I177605424","https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":8.9726,"has_fulltext":false,"cited_by_count":47,"citation_normalized_percentile":{"value":0.98180121,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"48","issue":"2","first_page":"637","last_page":"661"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11005","display_name":"Radiation Effects in Electronics","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11032","display_name":"VLSI and Analog Circuit Testing","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8313000202178955},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5062453150749207},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4876910448074341},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.42076370120048523}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8313000202178955},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5062453150749207},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4876910448074341},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.42076370120048523},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tse.2020.2998785","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2020.2998785","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.550000011920929}],"awards":[{"id":"https://openalex.org/G2729628976","display_name":null,"funder_award_id":"CCF-1750116","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4088831233","display_name":null,"funder_award_id":"CCF-1563797","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6112612032","display_name":null,"funder_award_id":"CCF-1453474","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8958528245","display_name":null,"funder_award_id":"CCF-1564162","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":146,"referenced_works":["https://openalex.org/W841012168","https://openalex.org/W1475493299","https://openalex.org/W1563891759","https://openalex.org/W1576818901","https://openalex.org/W1693912443","https://openalex.org/W1710734607","https://openalex.org/W1963523113","https://openalex.org/W1967959506","https://openalex.org/W1971991620","https://openalex.org/W1975455521","https://openalex.org/W1977696633","https://openalex.org/W1978232027","https://openalex.org/W1986544324","https://openalex.org/W1989545482","https://openalex.org/W1991097419","https://openalex.org/W1993760289","https://openalex.org/W1996832037","https://openalex.org/W2007777090","https://openalex.org/W2008626182","https://openalex.org/W2016027000","https://openalex.org/W2017665849","https://openalex.org/W2023930116","https://openalex.org/W2025791343","https://openalex.org/W2028967922","https://openalex.org/W2029726540","https://openalex.org/W2030857403","https://openalex.org/W2036740020","https://openalex.org/W2039532210","https://openalex.org/W2039772616","https://openalex.org/W2047093659","https://openalex.org/W2060333670","https://openalex.org/W2061575154","https://openalex.org/W2063171995","https://openalex.org/W2063387237","https://openalex.org/W2064232673","https://openalex.org/W2067436653","https://openalex.org/W2070544288","https://openalex.org/W2076719273","https://openalex.org/W2092382400","https://openalex.org/W2093101674","https://openalex.org/W2096713153","https://openalex.org/W2096795881","https://openalex.org/W2097174997","https://openalex.org/W2098549348","https://openalex.org/W2099866050","https://openalex.org/W2104107939","https://openalex.org/W2105899810","https://openalex.org/W2107709519","https://openalex.org/W2110706065","https://openalex.org/W2114503684","https://openalex.org/W2118655104","https://openalex.org/W2122947685","https://openalex.org/W2126071695","https://openalex.org/W2126105956","https://openalex.org/W2126752493","https://openalex.org/W2128705399","https://openalex.org/W2130877788","https://openalex.org/W2132525863","https://openalex.org/W2132852010","https://openalex.org/W2135937266","https://openalex.org/W2138428785","https://openalex.org/W2142406085","https://openalex.org/W2145124323","https://openalex.org/W2145373440","https://openalex.org/W2146136779","https://openalex.org/W2146889851","https://openalex.org/W2147050153","https://openalex.org/W2147524598","https://openalex.org/W2149783794","https://openalex.org/W2150990363","https://openalex.org/W2151497118","https://openalex.org/W2153881107","https://openalex.org/W2156553998","https://openalex.org/W2156723666","https://openalex.org/W2174442795","https://openalex.org/W2274071363","https://openalex.org/W2288083000","https://openalex.org/W2295100577","https://openalex.org/W2344973853","https://openalex.org/W2373227884","https://openalex.org/W2465133314","https://openalex.org/W2466388701","https://openalex.org/W2471601946","https://openalex.org/W2486590439","https://openalex.org/W2518136680","https://openalex.org/W2522171976","https://openalex.org/W2537787699","https://openalex.org/W2605202003","https://openalex.org/W2608579592","https://openalex.org/W2620986014","https://openalex.org/W2730550703","https://openalex.org/W2735107749","https://openalex.org/W2735571786","https://openalex.org/W2740130372","https://openalex.org/W2740264376","https://openalex.org/W2740646481","https://openalex.org/W2741422284","https://openalex.org/W2762550985","https://openalex.org/W2767431443","https://openalex.org/W2767766265","https://openalex.org/W2787135123","https://openalex.org/W2791415964","https://openalex.org/W2794443436","https://openalex.org/W2794832431","https://openalex.org/W2795030435","https://openalex.org/W2795607463","https://openalex.org/W2795866244","https://openalex.org/W2798812031","https://openalex.org/W2801865415","https://openalex.org/W2804618777","https://openalex.org/W2867448323","https://openalex.org/W2879105418","https://openalex.org/W2887430134","https://openalex.org/W2889801262","https://openalex.org/W2898851569","https://openalex.org/W2940501901","https://openalex.org/W2952920225","https://openalex.org/W2953998535","https://openalex.org/W2955624337","https://openalex.org/W2962708851","https://openalex.org/W2962809971","https://openalex.org/W2963058168","https://openalex.org/W2966980041","https://openalex.org/W2967591898","https://openalex.org/W2972704946","https://openalex.org/W2979679630","https://openalex.org/W2991598122","https://openalex.org/W3100600242","https://openalex.org/W3105092369","https://openalex.org/W3106103749","https://openalex.org/W3106729728","https://openalex.org/W3206322347","https://openalex.org/W4210484371","https://openalex.org/W4231241365","https://openalex.org/W4236837729","https://openalex.org/W4237412827","https://openalex.org/W4240124504","https://openalex.org/W4244051999","https://openalex.org/W4244452926","https://openalex.org/W4254753190","https://openalex.org/W4255632703","https://openalex.org/W4255726755","https://openalex.org/W4256216601","https://openalex.org/W4256377773","https://openalex.org/W6608438711","https://openalex.org/W6637688222"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Automated":[0],"program":[1,25,102,194],"repair":[2,26,46,72,103,333],"is":[3,117,219],"a":[4,192,197,222,228,315,320],"promising":[5],"approach":[6],"to":[7,31,38,44,50,52,97,123,132,251,281,287],"reducing":[8],"the":[9,39,45,53,67,83,114,133,145,165,168,205,208,216,232,235,246,294,329],"costs":[10],"of":[11,34,41,69,113,144,151,167,189,207,234,257,274,298,301,322,331],"manual":[12],"debugging":[13],"and":[14,48,77,110,141,285,292,313],"increasing":[15],"software":[16],"quality.":[17],"However,":[18],"recent":[19],"studies":[20],"have":[21],"shown":[22],"that":[23,191],"automated":[24,101],"techniques":[27,104,269,334],"can":[28],"be":[29],"prone":[30],"producing":[32,174,241],"patches":[33,106,129,146,247],"low":[35],"quality,":[36],"overfitting":[37],"set":[40,150],"tests":[42,190,239],"provided":[43,134],"technique,":[47],"failing":[49],"generalize":[51],"intended":[54],"specification.":[55],"This":[56],"paper":[57],"rigorously":[58],"explores":[59],"this":[60],"phenomenon":[61],"on":[62,79,164,204,231,289,335],"real-world":[63,98,336],"Java":[64,99,275,290,332],"programs,":[65],"analyzing":[66],"effectiveness":[68],"four":[70],"well-known":[71],"techniques,":[73],"GenProg,":[74],"Par,":[75],"SimFix,":[76],"TrpAutoRepair,":[78],"defects":[80],"made":[81],"by":[82,221],"projects\u2019":[84],"developers":[85],"during":[86],"their":[87],"regular":[88],"development":[89],"process.":[90],"We":[91,261],"find":[92],"that:":[93],"(1)":[94],"When":[95],"applied":[96,122],"code,":[100,291],"produce":[105],"for":[107,267,270,310,326],"between":[108,139],"10.6":[109],"19.0":[111],"percent":[112,143],"defects,":[115],"which":[116],"less":[118],"frequent":[119],"than":[120],"when":[121],"C":[124],"code.":[125],"(2)":[126],"The":[127,187],"produced":[128,209],"often":[130],"overfit":[131],"test":[135,172,180,217,324],"suite,":[136],"with":[137,170,183,237],"only":[138],"13.8":[140],"46.1":[142],"passing":[147],"an":[148,158,264,299],"independent":[149],"tests.":[152],"(3)":[153],"Test":[154,212],"suite":[155,213,218],"size":[156],"has":[157,196,227],"extremely":[159],"small":[160,198],"but":[161,199],"significant":[162,201,229],"effect":[163,203,230],"quality":[166,206,233,253,330],"patches,":[169,176,236],"larger":[171],"suites":[173,181],"higher-quality":[175,242],"though,":[177],"surprisingly,":[178],"higher-coverage":[179],"correlate":[182],"lower-quality":[184],"patches.":[185,210,243,260],"(4)":[186],"number":[188],"buggy":[193],"fails":[195],"statistically":[200],"positive":[202],"(5)":[211],"provenance,":[214],"whether":[215],"written":[220],"human":[223],"or":[224],"automatically":[225],"generated,":[226],"developer-written":[238],"typically":[240],"And":[244],"(6)":[245],"exhibit":[248],"insufficient":[249],"diversity":[250],"improve":[252],"through":[254],"some":[255],"method":[256],"combining":[258],"multiple":[259],"develop":[262],"JaRFly,":[263],"open-source":[265],"framework":[266],"implementing":[268],"automatic":[271],"search-based":[272],"improvement":[273],"programs.":[276],"Our":[277],"study":[278,307],"uses":[279],"JaRFly":[280],"faithfully":[282],"reimplement":[283],"GenProg":[284],"TrpAutoRepair":[286],"work":[288],"makes":[293],"first":[295],"public":[296],"release":[297],"implementation":[300],"Par.":[302],"Unlike":[303],"prior":[304],"work,":[305],"our":[306],"carefully":[308],"controls":[309],"confounding":[311],"factors":[312],"produces":[314],"methodology,":[316],"as":[317,319],"well":[318],"dataset":[321],"automatically-generated":[323],"suites,":[325],"objectively":[327],"evaluating":[328],"defects.":[337]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":13},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":3}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}