{"id":"https://openalex.org/W7143378675","doi":"https://doi.org/10.48550/arxiv.2603.26130","title":"SWE-PRBench: Benchmarking AI Code Review Quality Against Pull Request Feedback","display_name":"SWE-PRBench: Benchmarking AI Code Review Quality Against Pull Request Feedback","publication_year":2026,"publication_date":"2026-03-27","ids":{"openalex":"https://openalex.org/W7143378675","doi":"https://doi.org/10.48550/arxiv.2603.26130"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.26130","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.26130","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.26130","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5130964033","display_name":"Deepak Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Kumar, Deepak","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5130964033"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5928999781608582,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.5928999781608582,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.04610000178217888,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.044599998742341995,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7674999833106995},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6765000224113464},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6115999817848206},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6064000129699707},{"id":"https://openalex.org/keywords/ground-truth","display_name":"Ground truth","score":0.4717000126838684},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.4235000014305115},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.39340001344680786}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8055999875068665},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7674999833106995},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6765000224113464},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6115999817848206},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6064000129699707},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.4717000126838684},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4235000014305115},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.39340001344680786},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3797999918460846},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.36820000410079956},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36809998750686646},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36489999294281006},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3540000021457672},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.319599986076355},{"id":"https://openalex.org/C199519371","wikidata":"https://www.wikidata.org/wiki/Q942695","display_name":"Source lines of code","level":3,"score":0.3086000084877014},{"id":"https://openalex.org/C2779346075","wikidata":"https://www.wikidata.org/wiki/Q7268763","display_name":"Quality Score","level":3,"score":0.2777999937534332},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.275299996137619},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.27160000801086426},{"id":"https://openalex.org/C128942645","wikidata":"https://www.wikidata.org/wiki/Q1568346","display_name":"Test case","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C168065819","wikidata":"https://www.wikidata.org/wiki/Q845566","display_name":"Debugging","level":2,"score":0.26429998874664307}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.26130","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.26130","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.26130","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.26130","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.7002235054969788}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"introduce":[1],"SWE-PRBench,":[2],"a":[3,71,131,146,152,180],"benchmark":[4],"of":[5,33,97,133],"350":[6],"pull":[7],"requests":[8,59],"with":[9,86,140,157],"human-annotated":[10],"ground":[11],"truth":[12],"for":[13],"evaluating":[14],"AI":[15,42],"code":[16,43,55],"review":[17,44],"quality.":[18],"Evaluated":[19],"against":[20],"an":[21],"LLM-as-judge":[22],"framework":[23],"validated":[24],"at":[25,137],"kappa=0.75,":[26],"8":[27,102,167],"frontier":[28],"models":[29,103,172],"detect":[30],"only":[31,83],"15-31%":[32],"human-flagged":[34],"issues":[35],"on":[36,54],"the":[37,187],"diff-only":[38],"configuration,":[39],"demonstrating":[40],"that":[41],"remains":[45],"far":[46],"below":[47],"human":[48],"expert":[49],"performance":[50],"despite":[51],"strong":[52],"results":[53],"generation":[56],"benchmarks.":[57],"Pull":[58],"are":[60,173,200],"drawn":[61],"from":[62,67,106,186],"active":[63],"open-source":[64],"repositories,":[65],"filtered":[66],"700":[68],"candidates":[69],"using":[70],"Repository":[72],"Quality":[73],"Score,":[74],"and":[75,90,123,162,197],"evaluated":[76],"under":[77],"three":[78],"frozen":[79],"context":[80,92,98,112,122],"configurations:":[81],"diff":[82,85],"(config_A),":[84],"file":[87],"content":[88],"(config_B),":[89],"full":[91],"(config_C),":[93],"enabling":[94],"systematic":[95],"ablation":[96],"provision":[99],"strategies.":[100],"All":[101],"degrade":[104],"monotonically":[105],"config_A":[107],"to":[108],"config_C,":[109],"even":[110],"when":[111],"is":[113,130],"provided":[114],"via":[115],"structured":[116,147],"semantic":[117],"layers":[118],"including":[119],"AST-extracted":[120],"function":[121],"import":[124],"graph":[125],"resolution.":[126],"The":[127,169],"dominant":[128],"mechanism":[129],"collapse":[132],"Type2_Contextual":[134],"issue":[135],"detection":[136],"config_B,":[138],"consistent":[139],"attention":[141],"dilution":[142],"in":[143],"long":[144],"contexts:":[145],"2,000-token":[148],"diff-with-summary":[149],"prompt":[150,155],"outperforms":[151],"2,500-token":[153],"full-context":[154],"enriched":[156],"execution":[158],"context,":[159],"behaviour":[160],"mapping,":[161],"test":[163],"signatures":[164],"across":[165],"all":[166],"models.":[168],"top":[170],"four":[171,189],"statistically":[174],"indistinguishable":[175],"(mean":[176,190],"score":[177,191],"0.147-0.153)":[178],"while":[179],"clear":[181],"tier":[182],"gap":[183],"separates":[184],"them":[185],"remaining":[188],"<=":[192],"0.113).":[193],"Dataset,":[194],"contexts,":[195],"annotations,":[196],"evaluation":[198],"harness":[199],"released":[201],"publicly.":[202]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-31T00:00:00"}