{"id":"https://openalex.org/W4410771641","doi":"https://doi.org/10.1145/3736405","title":"Improving Code Reviewer Recommendation: Accuracy, Latency, Workload, and Bystanders","display_name":"Improving Code Reviewer Recommendation: Accuracy, Latency, Workload, and Bystanders","publication_year":2025,"publication_date":"2025-05-27","ids":{"openalex":"https://openalex.org/W4410771641","doi":"https://doi.org/10.1145/3736405"},"language":"en","primary_location":{"id":"doi:10.1145/3736405","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3736405","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020289098","display_name":"Peter C. Rigby","orcid":"https://orcid.org/0000-0003-1137-4297"},"institutions":[{"id":"https://openalex.org/I60158472","display_name":"Concordia University","ror":"https://ror.org/0420zvk78","country_code":"CA","type":"education","lineage":["https://openalex.org/I60158472"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Peter C. Rigby","raw_affiliation_strings":["Department of Computer Science and Software Engineering, Concordia University, Montreal, Quebec, Canada\u00a0and Meta Platforms Inc, Menlo\u00a0Park, California, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Software Engineering, Concordia University, Montreal, Quebec, Canada\u00a0and Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I60158472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102067625","display_name":"Seth Rogers","orcid":"https://orcid.org/0009-0003-1587-0578"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Seth Rogers","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093618432","display_name":"Sadruddin Saleem","orcid":"https://orcid.org/0009-0008-2848-3442"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sadruddin Saleem","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101286328","display_name":"Parth Suresh","orcid":"https://orcid.org/0000-0002-0087-8759"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Parth Suresh","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093618433","display_name":"Daniel Suskin","orcid":"https://orcid.org/0009-0001-0710-7906"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Suskin","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093383204","display_name":"Patrick Riggs","orcid":"https://orcid.org/0009-0008-9549-8695"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Patrick Riggs","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088100364","display_name":"Chandra Maddila","orcid":"https://orcid.org/0000-0002-9432-1045"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chandra Maddila","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101612061","display_name":"Nachiappan Nagappan","orcid":"https://orcid.org/0000-0003-1358-4124"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nachiappan Nagappan","raw_affiliation_strings":["Meta Platforms Inc, Menlo\u00a0Park, California, USA","Meta, USA"],"affiliations":[{"raw_affiliation_string":"Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I4210114444"]},{"raw_affiliation_string":"Meta, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5067199285","display_name":"Audris Mockus","orcid":"https://orcid.org/0000-0002-7987-7598"},"institutions":[{"id":"https://openalex.org/I2802706902","display_name":"Knoxville College","ror":"https://ror.org/02bxrp522","country_code":"US","type":"education","lineage":["https://openalex.org/I2802706902"]},{"id":"https://openalex.org/I75027704","display_name":"University of Tennessee at Knoxville","ror":"https://ror.org/020f3ap87","country_code":"US","type":"education","lineage":["https://openalex.org/I75027704"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Audris Mockus","raw_affiliation_strings":["The University of Tennessee, Knoxville, Knoxville, Tennessee, USA\u00a0and Meta Platforms Inc, Menlo\u00a0Park, California, USA"],"affiliations":[{"raw_affiliation_string":"The University of Tennessee, Knoxville, Knoxville, Tennessee, USA\u00a0and Meta Platforms Inc, Menlo\u00a0Park, California, USA","institution_ids":["https://openalex.org/I75027704","https://openalex.org/I2802706902"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5020289098"],"corresponding_institution_ids":["https://openalex.org/I60158472"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.18211782,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"35","issue":"1","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8255027532577515},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.7057055830955505},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5628275275230408},{"id":"https://openalex.org/keywords/latency","display_name":"Latency (audio)","score":0.5104771852493286},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.19654083251953125},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.1365036964416504},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12517637014389038}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8255027532577515},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.7057055830955505},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5628275275230408},{"id":"https://openalex.org/C82876162","wikidata":"https://www.wikidata.org/wiki/Q17096504","display_name":"Latency (audio)","level":2,"score":0.5104771852493286},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.19654083251953125},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.1365036964416504},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12517637014389038},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3736405","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3736405","pdf_url":null,"source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":36,"referenced_works":["https://openalex.org/W1487764496","https://openalex.org/W1511267087","https://openalex.org/W1576040511","https://openalex.org/W1982335236","https://openalex.org/W1998900885","https://openalex.org/W2006192515","https://openalex.org/W2008946935","https://openalex.org/W2049736950","https://openalex.org/W2091990486","https://openalex.org/W2099332975","https://openalex.org/W2111199933","https://openalex.org/W2115584760","https://openalex.org/W2138392083","https://openalex.org/W2145890718","https://openalex.org/W2148854374","https://openalex.org/W2151979607","https://openalex.org/W2161407365","https://openalex.org/W2167117640","https://openalex.org/W2169232226","https://openalex.org/W2288177242","https://openalex.org/W2344367508","https://openalex.org/W2374812233","https://openalex.org/W2395791174","https://openalex.org/W2399562897","https://openalex.org/W2413171555","https://openalex.org/W2567878593","https://openalex.org/W2808429234","https://openalex.org/W2810913898","https://openalex.org/W2959310500","https://openalex.org/W3090548469","https://openalex.org/W3091115553","https://openalex.org/W3103880005","https://openalex.org/W4308643081","https://openalex.org/W4365999169","https://openalex.org/W4383898351","https://openalex.org/W4392121835"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2000785801","https://openalex.org/W986318368","https://openalex.org/W2384410913","https://openalex.org/W2352878646","https://openalex.org/W2004734601","https://openalex.org/W2287264064","https://openalex.org/W2130149817"],"abstract_inverted_index":{"Aim":[0],".":[1,32,126,209,263,328,390,449],"The":[2,51,176,191,427],"code":[3,12,28,56],"review":[4,13,342,385],"team":[5,335,372,444],"at":[6,324,330],"Meta":[7,331],"is":[8,53,212,493],"continuously":[9],"improving":[10],"the":[11,35,55,65,72,107,120,143,166,182,197,200,221,228,239,283,302,313,316,320,347,364,379,386,408,452,508,526],"process.":[14],"In":[15,153],"this":[16,434],"work,":[17,457],"we":[18,37,81,90,163,234,374],"report":[19],"on":[20,119,135,157,227,251,258,297,396,519],"three":[21,39,381],"randomized":[22,47],"controlled":[23,48],"experimental":[24,49,73,472],"trials":[25],"to":[26,83,94,99,195,203,219,237,268,306,341,384,416,432,437,481],"improve":[27],"reviewer":[29,243,275,288,340],"recommendation.":[30],"Method":[31],"To":[33],"evaluate":[34],"recommenders,":[36],"conduct":[38],"A/B":[40,155,249,265,394,470,477],"tests":[41,478],"which":[42],"are":[43,479,490],"a":[44,61,110,129,186,259,272,308,334,343,371,404,443,464,501,512,516],"type":[45,121],"of":[46,122,184,205,223,242,301,304,355,378,410,528],"trial.":[50],"unit":[52],"either":[54],"diff":[57,187,253,387],"(Meta\u2019s":[58],"term":[59],"for":[60,414,446,533],"pull-request)":[62],"or":[63,115],"all":[64,204,438],"diffs":[66,159,357,367,415,439],"that":[67,89,137,146,165,368,423,440,475,507],"an":[68,154,248,338,393,530],"author":[69],"creates":[70],"during":[71],"period.":[74],"We":[75,105,127,245,345,391,419,504],"set":[76],"goal":[77,217],"metrics,":[78,87],"i.e.,":[79,97],"those":[80,88],"expect":[82],"improve,":[84],"and":[85,145,172,230,402,469,474],"guardrail":[86,299],"do":[91],"not":[92,180,213,291,421],"want":[93],"negatively":[95],"impact,":[96],"analogous":[98],"safety":[100],"metrics":[101,300],"in":[102,142,160,199,255,322,399,407,484,496,510],"medical":[103],"trials.":[104],"test":[106,117,156,250,266,395,471,482],"outcomes":[108],"using":[109,231],"t":[111],"-test,":[112],"Wilcoxon":[113],"test,":[114],"Fisher":[116],"depending":[118],"data.":[123],"Expt.":[124,207,326],"1":[125],"developed":[128],"new":[130,167,177],"recommender,":[131,261],"RevRecV2":[132],",":[133],"based":[134],"features":[136],"had":[138,173,276,370],"been":[139],"successfully":[140],"used":[141],"literature":[144,229],"could":[147],"be":[148,351,417,463],"calculated":[149],"with":[150,388],"low":[151,273,286],"latency.":[152,175],"82k":[158],"Spring":[161,400],"2022,":[162],"found":[164,403],"recommender":[168,178,198,314,321,435],"was":[169,188,218,289,294,362],"more":[170,476],"accurate":[171],"lower":[174],"did":[179,420],"impact":[181,296,518],"amount":[183,303,409],"time":[185,305,411],"under":[189],"review.":[190,309,365,447],"results":[192,428],"allowed":[193],"us":[194],"roll-out":[196],"Summer":[201],"2022":[202],"Meta.":[206,325],"2":[208],"Reviewer":[210],"workload":[211,222,274,287],"evenly":[214],"distributed,":[215],"our":[216,298,456,458],"reduce":[220],"top":[224,284,380],"reviewers.":[225],"Based":[226],"historical":[232,467],"data,":[233],"conducted":[235,392],"backtests":[236],"determine":[238],"best":[240],"measure":[241],"workload.":[244,503],"then":[246],"ran":[247],"28k":[252],"authors":[254,279,398,522],"Winter":[256],"2023":[257,401],"workload-balanced":[260],"RevRecWL":[262],"Our":[264],"led":[267],"mixed":[269],"results.":[270],"When":[271],"reasonable":[277],"expertise,":[278],"selected":[280,376],"them,":[281],"however,":[282],"recommended":[285,382],"often":[290,332,521],"selected.":[292],"There":[293],"no":[295,359],"perform":[307],"This":[310,492],"workload-balancing":[311],"replaced":[312],"from":[315,451,455],"first":[317],"experiment":[318],"as":[319],"production":[323],"3":[327],"Engineers":[329],"select":[333,523],"rather":[336],"than":[337],"individual":[339,361],"diff.":[344],"suspected":[346],"bystander":[348],"effect":[349],"might":[350],"slowing":[352],"down":[353],"reviews":[354],"these":[356],"because":[358],"single":[360],"assigned":[363,445],"On":[366],"only":[369,441],"assigned,":[373],"randomly":[375],"one":[377],"reviewers":[383,424],"BystanderRecRnd":[389],"12.5k":[397],"large":[405,517],"decrease":[406],"it":[412],"took":[413],"reviewed.":[418],"find":[422],"rushed":[425],"reviews.":[426],"were":[429],"strong":[430],"enough":[431],"roll":[433],"out":[436],"have":[442,515],"Implications":[448],"Aside":[450],"direct":[453],"findings":[454,459],"suggest":[460],"there":[461],"can":[462,514],"discrepancy":[465],"between":[466],"backtesting":[468],"findings,":[473],"necessary":[480],"recommenders":[483,499],"production.":[485],"Outcome":[486],"measures":[487],"beyond":[488],"accuracy":[489],"important.":[491],"especially":[494],"true":[495],"understanding":[497],"how":[498,520],"change":[500],"reviewer\u2019s":[502],"also":[505],"see":[506],"latency":[509,529],"displaying":[511],"recommendation":[513],"recommendations":[524],"making":[525],"reporting":[527],"important":[531],"metric":[532],"future":[534],"work.":[535]},"counts_by_year":[],"updated_date":"2025-12-13T23:11:00.310470","created_date":"2025-10-10T00:00:00"}