{"id":"https://openalex.org/W7172017512","doi":"https://doi.org/10.1145/3793302.3793614","title":"From Industry Claims to Empirical Reality: An Empirical Study of Code Review Agents in Pull Requests","display_name":"From Industry Claims to Empirical Reality: An Empirical Study of Code Review Agents in Pull Requests","publication_year":2026,"publication_date":"2026-04-13","ids":{"openalex":"https://openalex.org/W7172017512","doi":"https://doi.org/10.1145/3793302.3793614"},"language":"en","primary_location":{"id":"doi:10.1145/3793302.3793614","doi":"https://doi.org/10.1145/3793302.3793614","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3793302.3793614","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd International Conference on Mining Software Repositories","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3793302.3793614","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5118234295","display_name":"Kowshik Chowdhury","orcid":"https://orcid.org/0009-0009-3173-9899","observed_orcids":["https://orcid.org/0009-0009-3173-9899"]},"institutions":[{"id":"https://openalex.org/I172980758","display_name":"Kennesaw State University","ror":"https://ror.org/00jeqjx33","country_code":"US","type":"education","lineage":["https://openalex.org/I172980758"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kowshik Chowdhury","raw_affiliation_strings":["Computer Science, Kennesaw State University, Marietta, GA, USA"],"raw_orcid":"https://orcid.org/0009-0009-3173-9899","affiliations":[{"raw_affiliation_string":"Computer Science, Kennesaw State University, Marietta, GA, USA","institution_ids":["https://openalex.org/I172980758"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132612390","display_name":"Dipayan Banik","orcid":"https://orcid.org/0009-0009-5665-3071","observed_orcids":["https://orcid.org/0009-0009-5665-3071"]},"institutions":[{"id":"https://openalex.org/I100301340","display_name":"Quanta Technology (United States)","ror":"https://ror.org/03symbb61","country_code":"US","type":"company","lineage":["https://openalex.org/I100301340"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dipayan Banik","raw_affiliation_strings":["Quanta Technology, Raleigh, North Carolina, USA"],"raw_orcid":"https://orcid.org/0009-0009-5665-3071","affiliations":[{"raw_affiliation_string":"Quanta Technology, Raleigh, North Carolina, USA","institution_ids":["https://openalex.org/I100301340"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132590670","display_name":"K M Ferdous","orcid":"https://orcid.org/0009-0001-4125-6809","observed_orcids":["https://orcid.org/0009-0001-4125-6809"]},"institutions":[{"id":"https://openalex.org/I172980758","display_name":"Kennesaw State University","ror":"https://ror.org/00jeqjx33","country_code":"US","type":"education","lineage":["https://openalex.org/I172980758"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"K M Ferdous","raw_affiliation_strings":["Computer Science, Kennesaw State University, Marietta, GA, USA"],"raw_orcid":"https://orcid.org/0009-0001-4125-6809","affiliations":[{"raw_affiliation_string":"Computer Science, Kennesaw State University, Marietta, GA, USA","institution_ids":["https://openalex.org/I172980758"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019761662","display_name":"Shazibul Islam Shamim","orcid":"https://orcid.org/0000-0001-8084-5123","observed_orcids":["https://orcid.org/0000-0001-8084-5123"]},"institutions":[{"id":"https://openalex.org/I172980758","display_name":"Kennesaw State University","ror":"https://ror.org/00jeqjx33","country_code":"US","type":"education","lineage":["https://openalex.org/I172980758"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shazibul Islam Shamim","raw_affiliation_strings":["Software Engineering and Game Design and Development, Kennesaw State University, Marietta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0001-8084-5123","affiliations":[{"raw_affiliation_string":"Software Engineering and Game Design and Development, Kennesaw State University, Marietta, GA, USA","institution_ids":["https://openalex.org/I172980758"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.4526,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.94967143,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"984","last_page":"988"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9667999744415283,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9667999744415283,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.03150000050663948,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pull-requests","display_name":"pull requests","score":0.7710000276565552},{"id":"https://openalex.org/keywords/autonomous-coding-agents","display_name":"autonomous coding agents","score":0.7649999856948853},{"id":"https://openalex.org/keywords/openai-codex","display_name":"OpenAI Codex","score":0.8920000195503235},{"id":"https://openalex.org/keywords/signal-to-noise-ratio","display_name":"signal-to-noise ratio","score":0.7950000166893005},{"id":"https://openalex.org/keywords/open-source-software-development","display_name":"open source software development","score":0.6079999804496765}],"concepts":[{"id":"https://openalex.org/C197129107","wikidata":"https://www.wikidata.org/wiki/Q1921621","display_name":"Merge (version control)","level":2,"score":0.767799973487854},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6539000272750854},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.6310999989509583},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.5645999908447266},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5091999769210815},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.45190000534057617},{"id":"https://openalex.org/C166052673","wikidata":"https://www.wikidata.org/wiki/Q83021","display_name":"Empirical evidence","level":2,"score":0.42730000615119934},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.42239999771118164},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.38510000705718994},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3813999891281128},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.37599998712539673},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.36649999022483826},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3625999987125397},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.31940001249313354},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.3154999911785126},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.29350000619888306},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.29030001163482666},{"id":"https://openalex.org/C2778069772","wikidata":"https://www.wikidata.org/wiki/Q2119819","display_name":"Code of conduct","level":2,"score":0.27810001373291016},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.27480000257492065},{"id":"https://openalex.org/C2777131256","wikidata":"https://www.wikidata.org/wiki/Q7810111","display_name":"Title III","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2612999975681305}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3793302.3793614","doi":"https://doi.org/10.1145/3793302.3793614","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3793302.3793614","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd International Conference on Mining Software Repositories","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2604.03196","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2604.03196","pdf_url":"https://arxiv.org/pdf/2604.03196","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2604.03196","doi":"https://doi.org/10.48550/arxiv.2604.03196","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03196","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3793302.3793614","doi":"https://doi.org/10.1145/3793302.3793614","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3793302.3793614","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd International Conference on Mining Software Repositories","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7172017512.pdf","grobid_xml":"https://content.openalex.org/works/W7172017512.grobid-xml"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W3061716355","https://openalex.org/W4225632195","https://openalex.org/W4398795735","https://openalex.org/W4400878080","https://openalex.org/W4412888458","https://openalex.org/W4413360326","https://openalex.org/W4414915885","https://openalex.org/W4415746259","https://openalex.org/W4416360710","https://openalex.org/W4417420254","https://openalex.org/W7104698776","https://openalex.org/W7123346758","https://openalex.org/W7125586934","https://openalex.org/W7125914875","https://openalex.org/W7134077757","https://openalex.org/W7159928099"],"related_works":["https://openalex.org/W7167427608","https://openalex.org/W7171946610","https://openalex.org/W7171938800","https://openalex.org/W7213516748","https://openalex.org/W7211891524","https://openalex.org/W7162323345","https://openalex.org/W7134077757","https://openalex.org/W7171928444","https://openalex.org/W7172002873","https://openalex.org/W7158194875"],"abstract_inverted_index":{"Autonomous":[0],"coding":[1],"agents":[2,30],"are":[3],"generating":[4],"code":[5,28,250],"at":[6],"an":[7],"unprecedented":[8],"scale,":[9],"with":[10,172,223],"OpenAI":[11],"Codex":[12],"alone":[13],"creating":[14],"over":[15],"400,000":[16],"pull":[17,77],"requests":[18],"(PRs)":[19],"in":[20,36,49,133,206],"two":[21],"months.":[22],"As":[23,56],"agentic":[24],"PR":[25,88,109],"volumes":[26],"increase,":[27],"review":[29,135,208],"(CRAs)":[31],"have":[32],"become":[33],"routine":[34],"gatekeepers":[35],"development":[37],"workflows.":[38],"Industry":[39],"reports":[40],"claim":[41],"that":[42,180,213,231,241],"CRAs":[43,107,196,214,232],"can":[44],"manage":[45],"80%":[46],"of":[47,62,94,121,182,194],"PRs":[48,132,147,158,170,185],"open":[50],"source":[51],"repositories":[52],"without":[53,215],"human":[54,216,238,242],"involvement.":[55],"a":[57,160],"result,":[58],"understanding":[59],"the":[60,118,188],"effectiveness":[61],"CRA":[63,84],"reviews":[64],"is":[65,97],"crucial":[66],"for":[67,246],"maintaining":[68],"developmental":[69],"workflows":[70],"and":[71,101,105,117,192,240,248],"preventing":[72],"wasted":[73],"effort":[74],"on":[75,82],"abandoned":[76],"requests.":[78],"However,":[79],"empirical":[80],"evidence":[81],"how":[83,106],"feedback":[85,221],"quality":[86,120],"affects":[87],"outcomes":[89],"remains":[90,244],"limited.":[91],"The":[92],"goal":[93],"this":[95],"paper":[96],"to":[98,148,155],"help":[99],"researchers":[100],"practitioners":[102],"understand":[103],"when":[104],"influence":[108],"merge":[110,162],"success":[111],"by":[112],"empirically":[113],"analyzing":[114],"reviewer":[115],"composition":[116],"signal":[119,190,199],"CRA-generated":[122],"comments.":[123],"From":[124],"AIDev\u2019s":[125],"19,450":[126],"PRs,":[127],"we":[128],"analyze":[129],"3,109":[130],"unique":[131],"Commented":[134],"state,":[136],"comparing":[137],"human-only":[138,169],"versus":[139],"CRA-only":[140,146,157,184],"reviews.":[141],"We":[142],"examine":[143],"98":[144],"closed":[145,183],"assess":[149],"whether":[150],"low":[151],"signal-to-noise":[152,177],"ratios":[153,200],"contribute":[154],"abandonment.":[156,175,225],"achieve":[159],"45.20%":[161],"rate,":[163],"23.17":[164],"percentage":[165],"points":[166],"lower":[167],"than":[168,236],"(68.37%),":[171],"significantly":[173],"higher":[174,224],"Our":[176],"analysis":[178],"reveals":[179],"60.2%":[181],"fall":[186],"into":[187],"0\u201330%":[189],"range,":[191],"12":[193],"13":[195],"exhibit":[197],"average":[198],"below":[201],"60%,":[202],"indicating":[203],"substantial":[204],"noise":[205],"automated":[207],"feedback.":[209],"These":[210],"findings":[211],"suggest":[212],"oversight":[217],"often":[218],"generate":[219],"low-signal":[220],"associated":[222],"For":[226],"practitioners,":[227],"our":[228],"results":[229],"indicate":[230],"should":[233],"augment":[234],"rather":[235],"replace":[237],"reviewers,":[239],"involvement":[243],"critical":[245],"effective":[247],"actionable":[249],"review.":[251]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-08-01T00:00:00"}