{"id":"https://openalex.org/W1997548934","doi":"https://doi.org/10.1145/2597073.2597082","title":"Modern code reviews in open-source projects: which problems do they fix?","display_name":"Modern code reviews in open-source projects: which problems do they fix?","publication_year":2014,"publication_date":"2014-05-20","ids":{"openalex":"https://openalex.org/W1997548934","doi":"https://doi.org/10.1145/2597073.2597082","mag":"1997548934"},"language":"en","primary_location":{"id":"doi:10.1145/2597073.2597082","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2597073.2597082","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th Working Conference on Mining Software Repositories","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repository.tudelft.nl/file/File_9eb1b79b-a0f2-4631-9e2c-d71d4081a138","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021746976","display_name":"Moritz Beller","orcid":"https://orcid.org/0000-0003-4852-0526"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Moritz Beller","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082720005","display_name":"Alberto Bacchelli","orcid":"https://orcid.org/0000-0003-0193-6823"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Alberto Bacchelli","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046237164","display_name":"Andy Zaidman","orcid":"https://orcid.org/0000-0003-2413-3935"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Andy Zaidman","raw_affiliation_strings":["Delft University of Technology, Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5085630369","display_name":"Elmar Juergens","orcid":"https://orcid.org/0000-0003-1692-6839"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elmar Juergens","raw_affiliation_strings":["CQSE, Germany"],"affiliations":[{"raw_affiliation_string":"CQSE, Germany","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5021746976"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":63.8081,"has_fulltext":true,"cited_by_count":252,"citation_normalized_percentile":{"value":0.99905219,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"202","last_page":"211"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.8442548513412476},{"id":"https://openalex.org/keywords/maintainability","display_name":"Maintainability","score":0.8118463754653931},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.733269453048706},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6784234642982483},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.6109527349472046},{"id":"https://openalex.org/keywords/intuition","display_name":"Intuition","score":0.5960442423820496},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.5947659611701965},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.586689829826355},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.5802291631698608},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5530864000320435},{"id":"https://openalex.org/keywords/static-program-analysis","display_name":"Static program analysis","score":0.513640820980072},{"id":"https://openalex.org/keywords/code-refactoring","display_name":"Code refactoring","score":0.5091957449913025},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5090162754058838},{"id":"https://openalex.org/keywords/kpi-driven-code-analysis","display_name":"KPI-driven code analysis","score":0.5076104402542114},{"id":"https://openalex.org/keywords/open-source-software","display_name":"Open source software","score":0.47239920496940613},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.44050663709640503},{"id":"https://openalex.org/keywords/code-smell","display_name":"Code smell","score":0.4239102602005005},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3754808008670807},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.33647477626800537},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.3185397982597351},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1766262650489807}],"concepts":[{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.8442548513412476},{"id":"https://openalex.org/C160713754","wikidata":"https://www.wikidata.org/wiki/Q1389965","display_name":"Maintainability","level":2,"score":0.8118463754653931},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.733269453048706},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6784234642982483},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6109527349472046},{"id":"https://openalex.org/C132010649","wikidata":"https://www.wikidata.org/wiki/Q189222","display_name":"Intuition","level":2,"score":0.5960442423820496},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.5947659611701965},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.586689829826355},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.5802291631698608},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5530864000320435},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.513640820980072},{"id":"https://openalex.org/C152752567","wikidata":"https://www.wikidata.org/wiki/Q116877","display_name":"Code refactoring","level":3,"score":0.5091957449913025},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5090162754058838},{"id":"https://openalex.org/C121957198","wikidata":"https://www.wikidata.org/wiki/Q14365593","display_name":"KPI-driven code analysis","level":5,"score":0.5076104402542114},{"id":"https://openalex.org/C2988343187","wikidata":"https://www.wikidata.org/wiki/Q1130645","display_name":"Open source software","level":3,"score":0.47239920496940613},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.44050663709640503},{"id":"https://openalex.org/C133237599","wikidata":"https://www.wikidata.org/wiki/Q2295111","display_name":"Code smell","level":5,"score":0.4239102602005005},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3754808008670807},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.33647477626800537},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.3185397982597351},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1766262650489807},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2597073.2597082","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2597073.2597082","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 11th Working Conference on Mining Software Repositories","raw_type":"proceedings-article"},{"id":"pmh:tud:oai:tudelft.nl:uuid:93c23271-d299-4f3f-883f-a42d94a97c55","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:93c23271-d299-4f3f-883f-a42d94a97c55","pdf_url":"https://repository.tudelft.nl/file/File_9eb1b79b-a0f2-4631-9e2c-d71d4081a138","source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MSR 2014: ICSE 2014 - 36th International Conference on Software Engineering, Hyderabad, India, 31 May-7 June 2014; Authors version","raw_type":"info:eu-repo/semantics/conferencepaper"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.662.9389","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.662.9389","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://flosshub.org/sites/flosshub.org/files/beller.pdf","raw_type":"text"}],"best_oa_location":{"id":"pmh:tud:oai:tudelft.nl:uuid:93c23271-d299-4f3f-883f-a42d94a97c55","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:93c23271-d299-4f3f-883f-a42d94a97c55","pdf_url":"https://repository.tudelft.nl/file/File_9eb1b79b-a0f2-4631-9e2c-d71d4081a138","source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"MSR 2014: ICSE 2014 - 36th International Conference on Software Engineering, Hyderabad, India, 31 May-7 June 2014; Authors version","raw_type":"info:eu-repo/semantics/conferencepaper"},"sustainable_development_goals":[{"score":0.6200000047683716,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G629491556","display_name":null,"funder_award_id":"(NWO)","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W1997548934.pdf","grobid_xml":"https://content.openalex.org/works/W1997548934.grobid-xml"},"referenced_works_count":60,"referenced_works":["https://openalex.org/W569969942","https://openalex.org/W594002582","https://openalex.org/W604429516","https://openalex.org/W1444168786","https://openalex.org/W1507539036","https://openalex.org/W1952428424","https://openalex.org/W1986736724","https://openalex.org/W1988320117","https://openalex.org/W2002705589","https://openalex.org/W2006192515","https://openalex.org/W2013788150","https://openalex.org/W2017521602","https://openalex.org/W2048304653","https://openalex.org/W2050264635","https://openalex.org/W2053154970","https://openalex.org/W2063325123","https://openalex.org/W2063337812","https://openalex.org/W2065099045","https://openalex.org/W2068127755","https://openalex.org/W2078241739","https://openalex.org/W2091990486","https://openalex.org/W2095824036","https://openalex.org/W2102443580","https://openalex.org/W2102853796","https://openalex.org/W2103720965","https://openalex.org/W2107086891","https://openalex.org/W2108161968","https://openalex.org/W2125204770","https://openalex.org/W2127166753","https://openalex.org/W2139971640","https://openalex.org/W2140270836","https://openalex.org/W2142245496","https://openalex.org/W2147617769","https://openalex.org/W2150290224","https://openalex.org/W2151077051","https://openalex.org/W2153644070","https://openalex.org/W2157657084","https://openalex.org/W2158535304","https://openalex.org/W2159796803","https://openalex.org/W2161407365","https://openalex.org/W2166955195","https://openalex.org/W2170923331","https://openalex.org/W2171262837","https://openalex.org/W2177797324","https://openalex.org/W2226728217","https://openalex.org/W2497205942","https://openalex.org/W2796936280","https://openalex.org/W2912296587","https://openalex.org/W3010209906","https://openalex.org/W3101964726","https://openalex.org/W4231588088","https://openalex.org/W4248559578","https://openalex.org/W4285719527","https://openalex.org/W4300718545","https://openalex.org/W6607018326","https://openalex.org/W6616379744","https://openalex.org/W6652027039","https://openalex.org/W6663088175","https://openalex.org/W6682371025","https://openalex.org/W6683473158"],"related_works":["https://openalex.org/W2946499381","https://openalex.org/W125071098","https://openalex.org/W3166181464","https://openalex.org/W2955607951","https://openalex.org/W4382562158","https://openalex.org/W4384026574","https://openalex.org/W1982871693","https://openalex.org/W2150625980","https://openalex.org/W2108523483","https://openalex.org/W2338725043"],"abstract_inverted_index":{"Code":[0,20],"review":[1,145,161,167],"is":[2,49],"the":[3,27,32,55,59,73,105,111,122,130,153,166,173,178,184,213,216,222],"manual":[4],"assessment":[5],"of":[6,26,46,54,107,134,144,152,180,215,224],"source":[7,65],"code":[8,28,91,205],"by":[9,158],"humans,":[10],"mainly":[11],"intended":[12],"to":[13,50,110,119,136,183,193,211],"identify":[14],"defects":[15],"and":[16,39,124,149,196,202],"quality":[17],"problems.":[18,138],"Modern":[19],"Review":[21],"(MCR),":[22],"a":[23,97,203],"lightweight":[24],"variant":[25],"inspections":[29],"investigated":[30,170],"since":[31],"1970s,":[33],"prevails":[34],"today":[35],"both":[36],"in":[37,78,89,114,121,165],"industry":[38,123],"open-source":[40],"software":[41],"(OSS)":[42],"systems.":[43,80],"The":[44],"objective":[45],"this":[47],"paper":[48],"increase":[51],"our":[52],"understanding":[53],"practical":[56],"benefits":[57],"that":[58,68,104,142,150,176,189],"MCR":[60,77,112,185],"process":[61,113],"produces":[62],"on":[63,221],"reviewed":[64,90],"code.":[66],"To":[67],"end,":[69],"we":[70,169],"empirically":[71],"explore":[72],"problems":[74],"fixed":[75],"through":[76],"OSS":[79,94,115],"We":[81,139,187],"manually":[82],"classified":[83],"over":[84],"1,400":[85],"changes":[86,108,154,181,195],"taking":[87],"place":[88],"from":[92,127],"two":[93],"projects":[95],"into":[96],"validated":[98],"categorization":[99],"scheme.":[100],"Surprisingly,":[101],"results":[102],"show":[103],"types":[106],"due":[109,182],"are":[116,147,155],"strikingly":[117],"similar":[118,131],"those":[120],"academic":[125],"systems":[126],"literature,":[128],"featuring":[129],"75:25":[132],"ratio":[133],"maintainability-related":[135],"functional":[137],"also":[140],"reveal":[141],"7\u201335%":[143],"comments":[146],"discarded":[148],"10\u201322%":[151],"not":[156],"triggered":[157],"an":[159],"explicit":[160],"comment.":[162],"Patterns":[163],"emerged":[164],"data;":[168],"them":[171],"revealing":[172],"technical":[174],"factors":[175],"influence":[177],"number":[179,223],"process.":[186],"found":[188],"bug-fixing":[190],"tasks":[191,197],"lead":[192],"fewer":[194],"with":[198],"more":[199,208],"altered":[200],"files":[201],"higher":[204],"churn":[206],"have":[207],"changes.":[209,225],"Contrary":[210],"intuition,":[212],"person":[214],"reviewer":[217],"had":[218],"no":[219],"impact":[220]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":21},{"year":2022,"cited_by_count":22},{"year":2021,"cited_by_count":28},{"year":2020,"cited_by_count":29},{"year":2019,"cited_by_count":22},{"year":2018,"cited_by_count":20},{"year":2017,"cited_by_count":32},{"year":2016,"cited_by_count":19},{"year":2015,"cited_by_count":23},{"year":2014,"cited_by_count":4}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}