{"id":"https://openalex.org/W2794494697","doi":"https://doi.org/10.1145/3180155.3180192","title":"When testing meets code review","display_name":"When testing meets code review","publication_year":2018,"publication_date":"2018-05-27","ids":{"openalex":"https://openalex.org/W2794494697","doi":"https://doi.org/10.1145/3180155.3180192","mag":"2794494697"},"language":"en","primary_location":{"id":"doi:10.1145/3180155.3180192","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3180155.3180192","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034025036","display_name":"Davide Spadini","orcid":"https://orcid.org/0000-0003-2997-1890"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Davide Spadini","raw_affiliation_strings":["Software Improvement Group, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Software Improvement Group, Delft, The Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033008187","display_name":"Maur\u00edcio Aniche","orcid":"https://orcid.org/0000-0002-8893-2835"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maur\u00edcio Aniche","raw_affiliation_strings":["Delft University of Technology, Delft, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038905934","display_name":"Margaret\u2010Anne Storey","orcid":"https://orcid.org/0000-0003-2278-2536"},"institutions":[{"id":"https://openalex.org/I212119943","display_name":"University of Victoria","ror":"https://ror.org/04s5mat29","country_code":"CA","type":"education","lineage":["https://openalex.org/I212119943"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Margaret-Anne Storey","raw_affiliation_strings":["University of Victoria, Victoria, BC, Canada"],"affiliations":[{"raw_affiliation_string":"University of Victoria, Victoria, BC, Canada","institution_ids":["https://openalex.org/I212119943"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103139734","display_name":"Magiel Bruntink","orcid":"https://orcid.org/0000-0002-6117-6347"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Magiel Bruntink","raw_affiliation_strings":["Software Improvement Group, Amsterdam, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Software Improvement Group, Amsterdam, The Netherlands","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082720005","display_name":"Alberto Bacchelli","orcid":"https://orcid.org/0000-0003-0193-6823"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Alberto Bacchelli","raw_affiliation_strings":["University of Zurich, Zurich, Switzerland"],"affiliations":[{"raw_affiliation_string":"University of Zurich, Zurich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5034025036"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":9.457,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":{"value":0.97967174,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"677","last_page":"687"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.754692554473877},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7247664332389832},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.6357636451721191},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.6206873655319214},{"id":"https://openalex.org/keywords/code-coverage","display_name":"Code coverage","score":0.5807201266288757},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.49115273356437683},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.48264819383621216},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4765803813934326},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.46414533257484436},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.43960049748420715},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.3989405333995819},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.35406243801116943},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2176850140094757}],"concepts":[{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.754692554473877},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7247664332389832},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.6357636451721191},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6206873655319214},{"id":"https://openalex.org/C53942775","wikidata":"https://www.wikidata.org/wiki/Q1211721","display_name":"Code coverage","level":3,"score":0.5807201266288757},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.49115273356437683},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.48264819383621216},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4765803813934326},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.46414533257484436},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.43960049748420715},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3989405333995819},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.35406243801116943},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2176850140094757},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3180155.3180192","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3180155.3180192","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International Conference on Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:www.zora.uzh.ch:152981","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Spadini, Davide; Aniche, Maur\u00edcio; Storey, Margaret-Anne; Bruntink, Magiel; Bacchelli, Alberto  (2018). When testing meets code review: why and how developers review tests.  In: ICSE-SEIP '18: Proceedings of the 40th International Conference on Software Engineering: Software Engineering in Practice, Gothenburg, Sweden, 27 June 2018 - 3 July 2018. Institute of Electrical and Electronics Engineers, 677-687.","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:tudelft.nl:uuid:256e7d56-352f-44ae-919b-97fad0eafe69","is_oa":true,"landing_page_url":"http://resolver.tudelft.nl/uuid:256e7d56-352f-44ae-919b-97fad0eafe69","pdf_url":"http://resolver.tudelft.nl/uuid:256e7d56-352f-44ae-919b-97fad0eafe69","source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"},{"id":"doi:10.5167/uzh-152981","is_oa":true,"landing_page_url":"https://doi.org/10.5167/uzh-152981","pdf_url":null,"source":{"id":"https://openalex.org/S7407051291","display_name":"Universit\u00e4t Z\u00fcrich, ZORA","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":""}],"best_oa_location":{"id":"pmh:oai:www.zora.uzh.ch:152981","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Spadini, Davide; Aniche, Maur\u00edcio; Storey, Margaret-Anne; Bruntink, Magiel; Bacchelli, Alberto  (2018). When testing meets code review: why and how developers review tests.  In: ICSE-SEIP '18: Proceedings of the 40th International Conference on Software Engineering: Software Engineering in Practice, Gothenburg, Sweden, 27 June 2018 - 3 July 2018. Institute of Electrical and Electronics Engineers, 677-687.","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1093674026","display_name":null,"funder_award_id":"agreement","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G1524347686","display_name":"Co- and post-translational processing of oligomeric P-type ATP- ases","funder_award_id":"42954","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G3807287621","display_name":null,"funder_award_id":"PP00P2 170529","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G4750148621","display_name":null,"funder_award_id":"PP00P2_170529","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G5035863935","display_name":"Data-driven Contemporary Code Review","funder_award_id":"170529","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G580951548","display_name":null,"funder_award_id":"PP00P2","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":44,"referenced_works":["https://openalex.org/W659360797","https://openalex.org/W788122210","https://openalex.org/W1556297068","https://openalex.org/W1914972138","https://openalex.org/W1984080818","https://openalex.org/W1993659806","https://openalex.org/W1995223857","https://openalex.org/W1997548934","https://openalex.org/W1998900885","https://openalex.org/W2005403724","https://openalex.org/W2010295655","https://openalex.org/W2020400699","https://openalex.org/W2024132711","https://openalex.org/W2028897749","https://openalex.org/W2036487649","https://openalex.org/W2041114254","https://openalex.org/W2064893403","https://openalex.org/W2091990486","https://openalex.org/W2106013472","https://openalex.org/W2107294940","https://openalex.org/W2108769867","https://openalex.org/W2114063681","https://openalex.org/W2115919801","https://openalex.org/W2129377409","https://openalex.org/W2133561941","https://openalex.org/W2137266998","https://openalex.org/W2143712427","https://openalex.org/W2147386665","https://openalex.org/W2148143831","https://openalex.org/W2150874999","https://openalex.org/W2158348370","https://openalex.org/W2249980257","https://openalex.org/W2465706857","https://openalex.org/W2543971965","https://openalex.org/W2547102260","https://openalex.org/W2563377808","https://openalex.org/W2581357332","https://openalex.org/W2629457583","https://openalex.org/W3010856131","https://openalex.org/W3148292708","https://openalex.org/W3210030168","https://openalex.org/W4236010622","https://openalex.org/W4281564584","https://openalex.org/W6735265098"],"related_works":["https://openalex.org/W2904997879","https://openalex.org/W2344367508","https://openalex.org/W2019257047","https://openalex.org/W1997548934","https://openalex.org/W2097696338","https://openalex.org/W20188161","https://openalex.org/W2969773072","https://openalex.org/W4380568682","https://openalex.org/W2999298589","https://openalex.org/W2997072589"],"abstract_inverted_index":{"Automated":[0],"testing":[1],"is":[2,18,53,68,160,175],"considered":[3,22],"an":[4,119],"essential":[5],"process":[6],"for":[7,56,196],"ensuring":[8,57],"software":[9,34,41],"quality.":[10],"However,":[11],"writing":[12],"and":[13,20,32,49,73,82,95,105,137,167,194,201],"maintaining":[14],"high-quality":[15],"test":[16,61,89,113,135,158],"code":[17,37,51,103,123,150],"challenging":[19],"frequently":[21],"of":[23,60,65,78,121,128,134,140,177,192,199],"secondary":[24],"importance.":[25],"For":[26],"production":[27,165],"code,":[28,136],"many":[29],"open":[30],"source":[31],"industrial":[33],"projects":[35],"employ":[36],"review,":[38],"a":[39,126,138,190],"well-established":[40],"quality":[42,59],"practice,":[43],"but":[44],"the":[45,58,132,169,172,178,197],"question":[46,72],"remains":[47],"whether":[48],"how":[50,110],"review":[52,112,133,151,173],"also":[54],"used":[55],"code.":[62,90],"The":[63,153],"aim":[64],"this":[66,71],"research":[67],"to":[69,74,87,98,146],"answer":[70],"increase":[75],"our":[76,186],"understanding":[77],"what":[79],"developers":[80,108,143,181],"think":[81],"do":[83],"when":[84],"it":[85],"comes":[86],"reviewing":[88,124,157,164],"We":[91],"conducted":[92],"both":[93],"quantitative":[94],"qualitative":[96],"methods":[97],"analyze":[99],"more":[100],"than":[101],"300,000":[102],"reviews,":[104],"interviewed":[106],"12":[107],"about":[109],"they":[111],"files.":[114],"This":[115],"work":[116],"resulted":[117],"in":[118,149],"overview":[120],"current":[122],"practices,":[125],"set":[127,139],"identified":[129],"obstacles":[130],"limiting":[131],"issues":[141,180],"that":[142,156,168],"would":[144],"like":[145],"see":[147],"improved":[148],"tools.":[152],"study":[154],"reveals":[155],"files":[159],"very":[161],"different":[162],"from":[163],"files,":[166],"navigation":[170],"within":[171],"itself":[174],"one":[176],"main":[179],"currently":[182],"face.":[183],"Based":[184],"on":[185],"findings,":[187],"we":[188],"propose":[189],"series":[191],"recommendations":[193],"suggestions":[195],"design":[198],"tools":[200],"future":[202],"research.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":3},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":5}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}