{"id":"https://openalex.org/W7125417321","doi":"https://doi.org/10.1145/3786583.3786851","title":"RovoDev Code Reviewer: A Large-Scale Online Evaluation of LLM-based Code Review Automation at Atlassian","display_name":"RovoDev Code Reviewer: A Large-Scale Online Evaluation of LLM-based Code Review Automation at Atlassian","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W7125417321","doi":"https://doi.org/10.1145/3786583.3786851"},"language":null,"primary_location":{"id":"doi:10.1145/3786583.3786851","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786583.3786851","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 48th International Conference on Software Engineering: Software Engineering in Practice","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786583.3786851","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5122115740","display_name":"Kla Tantithamthavorn","orcid":null},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"education","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Kla Tantithamthavorn","raw_affiliation_strings":["Monash University, Melbourne, Australia and Atlassian, Melbourne, Australia"],"raw_orcid":"https://orcid.org/0000-0002-5516-9984","affiliations":[{"raw_affiliation_string":"Monash University, Melbourne, Australia and Atlassian, Melbourne, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122208403","display_name":"Yaotian Zou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098485","display_name":"Atlassian (United Kingdom)","ror":"https://ror.org/00y7n3708","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210098485"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yaotian Zou","raw_affiliation_strings":["Atlassian, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0001-9699-0429","affiliations":[{"raw_affiliation_string":"Atlassian, Seattle, USA","institution_ids":["https://openalex.org/I4210098485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123574107","display_name":"Andy Wong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andy Wong","raw_affiliation_strings":["Atlassian, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0009-7426-204X","affiliations":[{"raw_affiliation_string":"Atlassian, Sydney, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103415079","display_name":"Michael Gupta","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098485","display_name":"Atlassian (United Kingdom)","ror":"https://ror.org/00y7n3708","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210098485"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Michael Gupta","raw_affiliation_strings":["Atlassian, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0002-4406-2556","affiliations":[{"raw_affiliation_string":"Atlassian, Seattle, USA","institution_ids":["https://openalex.org/I4210098485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123565868","display_name":"Zhe Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhe Wang","raw_affiliation_strings":["Atlassian, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0003-8179-3347","affiliations":[{"raw_affiliation_string":"Atlassian, Sydney, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119913998","display_name":"Mike Buller","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mike Buller","raw_affiliation_strings":["Atlassian, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0007-7856-7882","affiliations":[{"raw_affiliation_string":"Atlassian, Sydney, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102448409","display_name":"Ryan Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryan Jiang","raw_affiliation_strings":["Atlassian, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0005-0262-0507","affiliations":[{"raw_affiliation_string":"Atlassian, Sydney, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122127970","display_name":"Matthew Watson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matthew Watson","raw_affiliation_strings":["Atlassian, Sydney, Australia"],"raw_orcid":"https://orcid.org/0009-0008-6525-7778","affiliations":[{"raw_affiliation_string":"Atlassian, Sydney, Australia","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123543703","display_name":"Minwoo Jeong","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098485","display_name":"Atlassian (United Kingdom)","ror":"https://ror.org/00y7n3708","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210098485"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Minwoo Jeong","raw_affiliation_strings":["Atlassian, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0008-9960-1741","affiliations":[{"raw_affiliation_string":"Atlassian, Seattle, USA","institution_ids":["https://openalex.org/I4210098485"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123565574","display_name":"Kun Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098485","display_name":"Atlassian (United Kingdom)","ror":"https://ror.org/00y7n3708","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210098485"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Kun Chen","raw_affiliation_strings":["Atlassian, Seattle, USA"],"raw_orcid":"https://orcid.org/0009-0009-1419-1788","affiliations":[{"raw_affiliation_string":"Atlassian, Seattle, USA","institution_ids":["https://openalex.org/I4210098485"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123562049","display_name":"Ming Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210098485","display_name":"Atlassian (United Kingdom)","ror":"https://ror.org/00y7n3708","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210098485"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ming Wu","raw_affiliation_strings":["Atlassian, Seattle, USA"],"raw_orcid":"https://orcid.org/0000-0002-6993-1431","affiliations":[{"raw_affiliation_string":"Atlassian, Seattle, USA","institution_ids":["https://openalex.org/I4210098485"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"34","last_page":"45"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8138999938964844,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8138999938964844,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.039500001817941666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.025599999353289604,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.7146999835968018},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.666100025177002},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.6340000033378601},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.5619000196456909},{"id":"https://openalex.org/keywords/kpi-driven-code-analysis","display_name":"KPI-driven code analysis","score":0.5108000040054321},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.41429999470710754},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4104999899864197}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7678999900817871},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.7146999835968018},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.666100025177002},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6556000113487244},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.6340000033378601},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.5619000196456909},{"id":"https://openalex.org/C121957198","wikidata":"https://www.wikidata.org/wiki/Q14365593","display_name":"KPI-driven code analysis","level":5,"score":0.5108000040054321},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.41429999470710754},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4104999899864197},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.362199991941452},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3569999933242798},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.3296999931335449},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3228999972343445},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.31459999084472656},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C151578736","wikidata":"https://www.wikidata.org/wiki/Q1251793","display_name":"Redundant code","level":4,"score":0.27950000762939453},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.2782999873161316},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.2540000081062317},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.25049999356269836}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3786583.3786851","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786583.3786851","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 48th International Conference on Software Engineering: Software Engineering in Practice","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2601.01129","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2601.01129","pdf_url":"https://arxiv.org/pdf/2601.01129","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3786583.3786851","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786583.3786851","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 48th International Conference on Software Engineering: Software Engineering in Practice","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)-powered":[3],"code":[4,12,20,32,54,71,111,118,126],"review":[5,13,21,33,55,72,112],"automation":[6,34,73],"has":[7],"the":[8,16,43,91,129,134,142,153],"potential":[9],"to":[10,117],"transform":[11],"workflows.":[14],"Despite":[15],"advances":[17],"of":[18,136,155],"LLM-powered":[19],"comment":[22,56],"generation":[23,57],"approaches,":[24],"several":[25],"practical":[26,44],"challenges":[27],"remain":[28],"for":[29,120],"designing":[30],"enterprise-grade":[31,69],"tools.":[35],"In":[36,60],"particular,":[37],"this":[38,61],"paper":[39],"aims":[40],"at":[41,78],"answering":[42],"question:":[45],"how":[46],"can":[47],"we":[48,63,101],"design":[49],"a":[50,98],"review-guided,":[51],"context-aware,":[52],"quality-checked":[53],"without":[58],"fine-tuning?":[59],"paper,":[62],"present":[64],"RovoDev":[65,104],"Code":[66,105],"Reviewer,":[67],"an":[68],"LLM-based":[70],"tool":[74],"designed":[75],"and":[76,132,160],"deployed":[77],"scale":[79],"within":[80],"Atlassian\u2019s":[81,88],"development":[82],"ecosystem":[83],"with":[84,168],"seamless":[85],"integration":[86],"into":[87],"Bitbucket.":[89],"Through":[90],"offline,":[92],"online,":[93],"user":[94],"feedback":[95,138],"evaluations":[96],"over":[97],"one-year":[99],"period,":[100],"conclude":[102],"that":[103,114,124],"Reviewer":[106],"is":[107],"effective":[108],"in":[109,128],"generating":[110],"comments":[113,123,157],"could":[115],"lead":[116],"resolution":[119],"38.70%":[121],"(i.e.,":[122,140,151,165],"triggered":[125],"changes":[127],"subsequent":[130],"commits);":[131],"offers":[133],"promise":[135],"accelerating":[137],"cycles":[139],"decreasing":[141],"PR":[143],"cycle":[144],"time":[145],"by":[146,158],"30.8%),":[147],"alleviating":[148],"reviewer":[149],"workload":[150],"reducing":[152],"number":[154],"human-written":[156],"35.6%),":[159],"improving":[161],"overall":[162],"software":[163],"quality":[164],"finding":[166],"errors":[167],"actionable":[169],"suggestions).":[170]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-01-23T00:00:00"}