{"id":"https://openalex.org/W4294558964","doi":"https://doi.org/10.1145/3561382","title":"Seeing the Whole Elephant: Systematically Understanding and Uncovering Evaluation Biases in Automated Program Repair","display_name":"Seeing the Whole Elephant: Systematically Understanding and Uncovering Evaluation Biases in Automated Program Repair","publication_year":2022,"publication_date":"2022-09-04","ids":{"openalex":"https://openalex.org/W4294558964","doi":"https://doi.org/10.1145/3561382"},"language":"en","primary_location":{"id":"doi:10.1145/3561382","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3561382","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003136837","display_name":"Deheng Yang","orcid":"https://orcid.org/0000-0002-8383-1939"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Deheng Yang","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100639534","display_name":"Yan Lei","orcid":"https://orcid.org/0000-0003-4504-6806"},"institutions":[{"id":"https://openalex.org/I158842170","display_name":"Chongqing University","ror":"https://ror.org/023rhb549","country_code":"CN","type":"education","lineage":["https://openalex.org/I158842170"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan Lei","raw_affiliation_strings":["Chongqing University, China"],"affiliations":[{"raw_affiliation_string":"Chongqing University, China","institution_ids":["https://openalex.org/I158842170"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080183182","display_name":"Xiaoguang Mao","orcid":"https://orcid.org/0000-0003-4204-7424"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoguang Mao","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081144020","display_name":"Yuhua Qi","orcid":"https://orcid.org/0000-0002-1246-3833"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuhua Qi","raw_affiliation_strings":["Beijing Institute of Tracking and Communication Technology, China"],"affiliations":[{"raw_affiliation_string":"Beijing Institute of Tracking and Communication Technology, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040799000","display_name":"Xin Yi","orcid":"https://orcid.org/0000-0003-4511-1495"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Yi","raw_affiliation_strings":["National University of Defense Technology, China"],"affiliations":[{"raw_affiliation_string":"National University of Defense Technology, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5003136837"],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":1.3743,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.80454404,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"32","issue":"3","first_page":"1","last_page":"37"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7894408106803894},{"id":"https://openalex.org/keywords/executable","display_name":"Executable","score":0.6956512928009033},{"id":"https://openalex.org/keywords/strengths-and-weaknesses","display_name":"Strengths and weaknesses","score":0.5230423212051392},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5212371945381165},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3282366394996643}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7894408106803894},{"id":"https://openalex.org/C160145156","wikidata":"https://www.wikidata.org/wiki/Q778586","display_name":"Executable","level":2,"score":0.6956512928009033},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.5230423212051392},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5212371945381165},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3282366394996643},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3561382","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3561382","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","score":0.6399999856948853,"display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G1627267444","display_name":null,"funder_award_id":"61872445 and 62272072","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":109,"referenced_works":["https://openalex.org/W841012168","https://openalex.org/W1514276232","https://openalex.org/W1857789879","https://openalex.org/W1963523113","https://openalex.org/W1969563312","https://openalex.org/W1991097419","https://openalex.org/W2005039125","https://openalex.org/W2023992080","https://openalex.org/W2025791343","https://openalex.org/W2049238280","https://openalex.org/W2061575154","https://openalex.org/W2063387237","https://openalex.org/W2066305013","https://openalex.org/W2088927456","https://openalex.org/W2104107939","https://openalex.org/W2106956101","https://openalex.org/W2110219362","https://openalex.org/W2122947685","https://openalex.org/W2128049346","https://openalex.org/W2138428785","https://openalex.org/W2145373440","https://openalex.org/W2149783794","https://openalex.org/W2153881107","https://openalex.org/W2156723666","https://openalex.org/W2168130919","https://openalex.org/W2168156367","https://openalex.org/W2244826483","https://openalex.org/W2285839903","https://openalex.org/W2343325785","https://openalex.org/W2343875716","https://openalex.org/W2344973853","https://openalex.org/W2400994325","https://openalex.org/W2465133314","https://openalex.org/W2512804022","https://openalex.org/W2518136680","https://openalex.org/W2574271863","https://openalex.org/W2582560208","https://openalex.org/W2620081107","https://openalex.org/W2754449345","https://openalex.org/W2762550985","https://openalex.org/W2772500824","https://openalex.org/W2794443436","https://openalex.org/W2795030435","https://openalex.org/W2795866244","https://openalex.org/W2801076109","https://openalex.org/W2804618777","https://openalex.org/W2809214741","https://openalex.org/W2809631772","https://openalex.org/W2850616187","https://openalex.org/W2851896161","https://openalex.org/W2867448323","https://openalex.org/W2883977877","https://openalex.org/W2890086692","https://openalex.org/W2895570420","https://openalex.org/W2904214673","https://openalex.org/W2905278195","https://openalex.org/W2905489084","https://openalex.org/W2908354350","https://openalex.org/W2910431138","https://openalex.org/W2924629359","https://openalex.org/W2933919840","https://openalex.org/W2945432765","https://openalex.org/W2950571912","https://openalex.org/W2953898913","https://openalex.org/W2953998535","https://openalex.org/W2954183584","https://openalex.org/W2954778170","https://openalex.org/W2958228760","https://openalex.org/W2963909831","https://openalex.org/W2965720427","https://openalex.org/W2967591898","https://openalex.org/W2968109196","https://openalex.org/W2973149860","https://openalex.org/W2975950293","https://openalex.org/W2979679630","https://openalex.org/W2998011150","https://openalex.org/W3000410628","https://openalex.org/W3003756849","https://openalex.org/W3009761259","https://openalex.org/W3012626099","https://openalex.org/W3016496139","https://openalex.org/W3043761819","https://openalex.org/W3084918652","https://openalex.org/W3088927226","https://openalex.org/W3089621332","https://openalex.org/W3090643686","https://openalex.org/W3100698844","https://openalex.org/W3102072242","https://openalex.org/W3104012431","https://openalex.org/W3121734253","https://openalex.org/W3134686229","https://openalex.org/W3135841251","https://openalex.org/W3137004499","https://openalex.org/W3156188980","https://openalex.org/W3160007051","https://openalex.org/W3160155705","https://openalex.org/W3161027892","https://openalex.org/W3161427495","https://openalex.org/W3178330452","https://openalex.org/W3206322347","https://openalex.org/W3208407575","https://openalex.org/W4231241365","https://openalex.org/W4232386622","https://openalex.org/W4251988601","https://openalex.org/W4255632703","https://openalex.org/W4280528532","https://openalex.org/W4288614128","https://openalex.org/W4291213652","https://openalex.org/W6833780757"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2350278424","https://openalex.org/W2071432835","https://openalex.org/W4234371507","https://openalex.org/W4299534542","https://openalex.org/W1628824497","https://openalex.org/W4239401009","https://openalex.org/W2053441600","https://openalex.org/W1990401748","https://openalex.org/W2047586841"],"abstract_inverted_index":{"Evaluation":[0],"is":[1,27,63],"the":[2,22,41,52,81,88,107,143,203],"foundation":[3],"of":[4,18,24,74,83,90,145,226],"automated":[5],"program":[6],"repair":[7],"(APR),":[8],"as":[9,216,218],"it":[10],"provides":[11],"empirical":[12],"evidence":[13],"on":[14,112,164,181],"strengths":[15],"and":[16,49,72,86,116,137,156,177,195,202,224],"weaknesses":[17],"APR":[19,42,57,91,179,229],"techniques.":[20,92],"However,":[21],"reliability":[23],"such":[25,84],"evaluation":[26,75,89,102,127],"often":[28],"threatened":[29],"by":[30,104,121],"various":[31],"introduced":[32],"biases.":[33,128],"Consequently,":[34],"bias":[35,141,192],"exploration,":[36],"which":[37,79,165],"uncovers":[38],"biases":[39,76,85,103,115,120,136,227],"in":[40,77,142,210,228],"evaluation,":[43],"has":[44],"become":[45],"a":[46,69,123,130,139],"pivotal":[47],"activity":[48],"performed":[50,201],"since":[51],"early":[53],"years":[54],"when":[55],"pioneer":[56],"techniques":[58,180],"were":[59],"proposed.":[60],"Unfortunately,":[61],"there":[62],"still":[64],"no":[65],"methodology":[66],"to":[67,98],"support":[68],"systematic":[70,109,198],"comprehension":[71],"discovery":[73],"APR,":[78],"impedes":[80],"mitigation":[82,225],"threatens":[87],"In":[93],"this":[94,151,186,211],"work,":[95],"we":[96,132,154,166,200,208],"propose":[97],"systematically":[99,117],"understand":[100],"existing":[101,113],"rigorously":[105],"conducting":[106],"first":[108],"literature":[110],"review":[111],"known":[114],"uncover":[118,138],"new":[119,140,152,214],"building":[122],"taxonomy":[124],"that":[125],"categorizes":[126],"As":[129],"result,":[131],"identify":[133],"17":[134],"investigated":[135],"usage":[144],"patch":[146,170],"validation":[147,171],"strategies.":[148],"To":[149],"validate":[150],"bias,":[153],"devise":[155],"implement":[157],"an":[158,219],"executable":[159,206],"framework":[160,207],"APRConfig":[161],",":[162],"based":[163],"evaluate":[167],"three":[168,182],"typical":[169],"strategies":[172],"with":[173],"four":[174],"representative":[175],"heuristic-based":[176],"constraint-based":[178],"bug":[183],"datasets.":[184],"Overall,":[185],"article":[187,212],"distills":[188],"13":[189],"findings":[190],"for":[191,221],"understanding,":[193],"discovery,":[194],"validation.":[196],"The":[197],"exploration":[199,223],"open":[204],"source":[205],"proposed":[209],"provide":[213],"insights":[215],"well":[217],"infrastructure":[220],"future":[222],"evaluation.":[230]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2}],"updated_date":"2026-03-22T08:09:32.410652","created_date":"2025-10-10T00:00:00"}