{"id":"https://openalex.org/W7140294178","doi":"https://doi.org/10.48550/arxiv.2603.23448","title":"Code Review Agent Benchmark","display_name":"Code Review Agent Benchmark","publication_year":2026,"publication_date":"2026-03-24","ids":{"openalex":"https://openalex.org/W7140294178","doi":"https://doi.org/10.48550/arxiv.2603.23448"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2603.23448","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23448","pdf_url":"https://arxiv.org/pdf/2603.23448","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.23448","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102920981","display_name":"Yuntong Zhang","orcid":"https://orcid.org/0009-0005-1664-7110","observed_orcids":["https://orcid.org/0009-0005-1664-7110"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Zhang, Yuntong","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130609825","display_name":"Zhiyuan Pan","orcid":"https://orcid.org/0009-0006-6059-5191","observed_orcids":["https://orcid.org/0009-0006-6059-5191"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pan, Zhiyuan","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062634807","display_name":"Imam Nur Bani Yusuf","orcid":"https://orcid.org/0000-0003-0900-5230","observed_orcids":["https://orcid.org/0000-0003-0900-5230"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yusuf, Imam Nur Bani","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101353985","display_name":"Haifeng Ruan","orcid":"https://orcid.org/0009-0008-1080-4770","observed_orcids":["https://orcid.org/0009-0008-1080-4770"]},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Ruan, Haifeng","raw_affiliation_strings":["National University of Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National University of Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052793365","display_name":"Ridwan Shariffdeen","orcid":"https://orcid.org/0000-0001-5409-4864","observed_orcids":["https://orcid.org/0000-0001-5409-4864"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shariffdeen, Ridwan","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5060115298","display_name":"Abhik Roychoudhury","orcid":"https://orcid.org/0000-0002-7127-1137","observed_orcids":["https://orcid.org/0000-0002-7127-1137"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roychoudhury, Abhik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark-dataset","display_name":"benchmark dataset","score":0.6809999942779541},{"id":"https://openalex.org/keywords/pull-requests","display_name":"pull requests","score":0.8460000157356262},{"id":"https://openalex.org/keywords/assessment-framework","display_name":"evaluation framework","score":0.4909999966621399},{"id":"https://openalex.org/keywords/human-agent-collaboration","display_name":"human-agent collaboration","score":0.6330000162124634},{"id":"https://openalex.org/keywords/test-suite-generation","display_name":"test suite generation","score":0.5120000243186951},{"id":"https://openalex.org/keywords/code-quality-assurance","display_name":"code quality assurance","score":0.6269999742507935},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5350000262260437},{"id":"https://openalex.org/keywords/codex","display_name":"codices","score":0.621999979019165},{"id":"https://openalex.org/keywords/claude-code","display_name":"Claude Code","score":0.8510000109672546},{"id":"https://openalex.org/keywords/devin","display_name":"Devin","score":0.4269999861717224}],"concepts":[{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.7315999865531921},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.7257999777793884},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7250999808311462},{"id":"https://openalex.org/C2776969324","wikidata":"https://www.wikidata.org/wiki/Q613918","display_name":"Software quality assurance","level":5,"score":0.6621000170707703},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5852000117301941},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.5325000286102295},{"id":"https://openalex.org/C133237599","wikidata":"https://www.wikidata.org/wiki/Q2295111","display_name":"Code smell","level":5,"score":0.5006999969482422},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.47200000286102295},{"id":"https://openalex.org/C2778956278","wikidata":"https://www.wikidata.org/wiki/Q392813","display_name":"Agent-oriented software engineering","level":4,"score":0.4625999927520752},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4456000030040741},{"id":"https://openalex.org/C10272871","wikidata":"https://www.wikidata.org/wiki/Q929972","display_name":"Software inspection","level":5,"score":0.42890000343322754},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.4235999882221222},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.4163999855518341},{"id":"https://openalex.org/C121957198","wikidata":"https://www.wikidata.org/wiki/Q14365593","display_name":"KPI-driven code analysis","level":5,"score":0.4090999960899353},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3709999918937683},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3702000081539154},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.36399999260902405},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.32330000400543213},{"id":"https://openalex.org/C103520596","wikidata":"https://www.wikidata.org/wiki/Q7554328","display_name":"Software mining","level":5,"score":0.3222000002861023},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29190000891685486},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.28839999437332153},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2865000069141388},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.28619998693466187},{"id":"https://openalex.org/C106436119","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assurance","level":3,"score":0.28369998931884766},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C2988963302","wikidata":"https://www.wikidata.org/wiki/Q629206","display_name":"Program code","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2603999972343445}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.23448","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23448","pdf_url":"https://arxiv.org/pdf/2603.23448","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.23448","doi":"https://doi.org/10.48550/arxiv.2603.23448","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.23448","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.23448","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.23448","pdf_url":"https://arxiv.org/pdf/2603.23448","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140294178.pdf","grobid_xml":"https://content.openalex.org/works/W7140294178.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W4399837631","https://openalex.org/W7125957520","https://openalex.org/W7126096038","https://openalex.org/W7128715130","https://openalex.org/W7135169507"],"related_works":["https://openalex.org/W7135169507","https://openalex.org/W4391590822","https://openalex.org/W7008713878","https://openalex.org/W7166908342","https://openalex.org/W7221505180","https://openalex.org/W7167688715","https://openalex.org/W4415937833","https://openalex.org/W4415179900","https://openalex.org/W7151672870","https://openalex.org/W7164565760"],"abstract_inverted_index":{"Software":[0],"engineering":[1],"agents":[2,12,74,86,102,149,199,293,297],"have":[3],"shown":[4],"significant":[5],"promise":[6],"in":[7,250],"writing":[8],"code.":[9],"As":[10,33],"AI":[11,73],"permeate":[13],"code":[14,21,27,37,47,69,88,100,107,123,147,182,244,288,295],"writing,":[15],"and":[16,31,49,66,154,272,294],"generate":[17,176],"huge":[18,41],"volumes":[19],"of":[20,26,46,121,135,170,207,287],"automatically":[22,35],"--":[23,43,139,165,236,298],"the":[24,34,44,64,118,122,133,136,140,181,196,208,212,225,233,238,257,259],"matter":[25],"quality":[28,51,274],"comes":[29],"front":[30],"centre.":[32],"generated":[36,185,261,278],"gets":[38],"integrated":[39],"into":[40],"code-bases":[42],"issue":[45],"review":[48,70,89,108,124,148,169,183,198,245,296],"broadly":[50],"assurance":[52],"becomes":[53],"important.":[54],"In":[55],"this":[56,216,281],"paper,":[57],"we":[58,175,222],"take":[59],"a":[60,68,93,106,111,167,171,188,268],"fresh":[61],"look":[62],"at":[63],"problem":[65],"curate":[67],"dataset":[71,79,158],"for":[72,87,240,243,276,284],"to":[75,131,179,214,300],"work":[76],"with.":[77],"Our":[78,126,156],"called":[80],"c-CRAB":[81,157,209],"(pronounced":[82],"see-crab)":[83],"can":[84,116,202],"evaluate":[85,132,180],"tasks.":[90],"Specifically":[91],"given":[92,166],"pull-request":[94],"(which":[95],"could":[96,247],"be":[97,248,301],"coming":[98],"from":[99,150,162,232,263],"generation":[101,289,292],"or":[103],"humans),":[104],"if":[105],"agent":[109,184,226,260,277],"produces":[110],"review,":[112],"our":[113,264],"evaluation":[114,127],"framework":[115,128],"asses":[117],"reviewing":[119],"capability":[120],"agents.":[125],"is":[129,159],"used":[130],"state":[134],"art":[137],"today":[138],"open-source":[141],"PR-agent,":[142],"as":[143,145,267],"well":[144],"commercial":[146],"Devin,":[151],"Claude":[152],"Code,":[153],"Codex.":[155],"systematically":[160],"constructed":[161],"human":[163,168,234],"reviews":[164,227,235],"pull":[172],"request":[173],"instance":[174],"corresponding":[177],"tests":[178,262],"reviews.":[186,279],"Such":[187],"benchmark":[189],"construction":[190],"gives":[191],"us":[192],"several":[193],"insights.":[194],"Firstly,":[195],"existing":[197],"taken":[200],"together":[201],"solve":[203],"only":[204],"around":[205],"40%":[206],"tasks,":[210],"indicating":[211,237],"potential":[213,239],"close":[215],"gap":[217],"by":[218],"future":[219,251,285],"research.":[220],"Secondly,":[221],"observe":[223],"that":[224,246],"often":[228],"consider":[229],"different":[230],"aspects":[231],"human-agent":[241],"collaboration":[242,286],"deployed":[249],"software":[252],"teams.":[253],"Last":[254],"but":[255],"not":[256],"least,":[258],"data-set":[265],"act":[266],"held":[269],"out":[270],"test-suite":[271],"hence":[273],"gate":[275],"What":[280],"will":[282],"mean":[283],"agents,":[290],"test":[291],"remains":[299],"investigated.":[302]},"counts_by_year":[{"year":2026,"cited_by_count":15}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-03-26T00:00:00"}