{"id":"https://openalex.org/W2603119212","doi":"https://doi.org/10.1109/access.2017.2685629","title":"Continuous Integration, Delivery and Deployment: A Systematic Review on Approaches, Tools, Challenges and Practices","display_name":"Continuous Integration, Delivery and Deployment: A Systematic Review on Approaches, Tools, Challenges and Practices","publication_year":2017,"publication_date":"2017-01-01","ids":{"openalex":"https://openalex.org/W2603119212","doi":"https://doi.org/10.1109/access.2017.2685629","mag":"2603119212"},"language":"en","primary_location":{"id":"doi:10.1109/access.2017.2685629","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2017.2685629","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"review","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2017.2685629","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052783352","display_name":"Mojtaba Shahin","orcid":"https://orcid.org/0000-0002-9081-1354"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Mojtaba Shahin","raw_affiliation_strings":["Centre for Research on Engineering Software Technologies, The University of Adelaide, Adelaide, SA, Australia"],"raw_orcid":"https://orcid.org/0000-0002-9081-1354","affiliations":[{"raw_affiliation_string":"Centre for Research on Engineering Software Technologies, The University of Adelaide, Adelaide, SA, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103075476","display_name":"Muhammad Ali Babar","orcid":"https://orcid.org/0000-0001-9696-3626"},"institutions":[{"id":"https://openalex.org/I5681781","display_name":"The University of Adelaide","ror":"https://ror.org/00892tw58","country_code":"AU","type":"education","lineage":["https://openalex.org/I5681781"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Muhammad Ali Babar","raw_affiliation_strings":["Centre for Research on Engineering Software Technologies, The University of Adelaide, Adelaide, SA, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centre for Research on Engineering Software Technologies, The University of Adelaide, Adelaide, SA, Australia","institution_ids":["https://openalex.org/I5681781"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064683660","display_name":"Liming Zhu","orcid":"https://orcid.org/0000-0001-5839-3765"},"institutions":[{"id":"https://openalex.org/I1292875679","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07","country_code":"AU","type":"government","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I4387156119"]},{"id":"https://openalex.org/I42894916","display_name":"Data61","ror":"https://ror.org/03q397159","country_code":"AU","type":"other","lineage":["https://openalex.org/I1292875679","https://openalex.org/I2801453606","https://openalex.org/I42894916","https://openalex.org/I4387156119"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Liming Zhu","raw_affiliation_strings":["Data61, Commonwealth Scientific and Industrial Research Organisation, Sydney, NSW, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Data61, Commonwealth Scientific and Industrial Research Organisation, Sydney, NSW, Australia","institution_ids":["https://openalex.org/I1292875679","https://openalex.org/I42894916"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5052783352"],"corresponding_institution_ids":["https://openalex.org/I5681781"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":64.8036,"has_fulltext":false,"cited_by_count":634,"citation_normalized_percentile":{"value":0.99893299,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"5","issue":null,"first_page":"3909","last_page":"3943"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6983658075332642},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.6907124519348145},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.6221354007720947},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.49031761288642883},{"id":"https://openalex.org/keywords/dependability","display_name":"Dependability","score":0.4628542959690094},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.45277953147888184},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.43843671679496765},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.38448530435562134},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.3371346890926361},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1859557032585144}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6983658075332642},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.6907124519348145},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.6221354007720947},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.49031761288642883},{"id":"https://openalex.org/C77019957","wikidata":"https://www.wikidata.org/wiki/Q2689057","display_name":"Dependability","level":2,"score":0.4628542959690094},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.45277953147888184},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.43843671679496765},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.38448530435562134},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3371346890926361},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1859557032585144},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":6,"locations":[{"id":"doi:10.1109/access.2017.2685629","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2017.2685629","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1703.07019","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1703.07019","pdf_url":"https://arxiv.org/pdf/1703.07019","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:alma61RMIT.INST:11294645380001341","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ACCESS.2017.2685629","pdf_url":null,"source":{"id":"https://openalex.org/S4306402074","display_name":"RMIT Research Repository (RMIT University Library)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I82951845","host_organization_name":"RMIT University","host_organization_lineage":["https://openalex.org/I82951845"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:digital.library.adelaide.edu.au:2440/116788","is_oa":false,"landing_page_url":"http://hdl.handle.net/2440/116788","pdf_url":null,"source":{"id":"https://openalex.org/S4306401835","display_name":"Adelaide Research & Scholarship (AR&S) (University of Adelaide)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I5681781","host_organization_name":"The University of Adelaide","host_organization_lineage":["https://openalex.org/I5681781"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://dx.doi.org/10.1109/access.2017.2685629","raw_type":"Journal article"},{"id":"pmh:oai:doaj.org/article:43c802475caa40f4b6d4345293e884f4","is_oa":true,"landing_page_url":"https://doaj.org/article/43c802475caa40f4b6d4345293e884f4","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 5, Pp 3909-3943 (2017)","raw_type":"article"},{"id":"pmh:oai:figshare.com:article/27556566","is_oa":true,"landing_page_url":"https://figshare.com/articles/journal_contribution/Continuous_Integration_Delivery_and_Deployment_A_Systematic_Review_on_Approaches_Tools_Challenges_and_Practices/27556566","pdf_url":null,"source":{"id":"https://openalex.org/S4377196282","display_name":"Figshare","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210132348","host_organization_name":"Figshare (United Kingdom)","host_organization_lineage":["https://openalex.org/I4210132348"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1109/access.2017.2685629","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2017.2685629","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.6100000143051147}],"awards":[],"funders":[{"id":"https://openalex.org/F4320315885","display_name":"Australian Government","ror":"https://ror.org/0314h5y94"},{"id":"https://openalex.org/F4320320386","display_name":"Commonwealth Scientific and Industrial Research Organisation","ror":"https://ror.org/03qn8fb07"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W159490162","https://openalex.org/W339001469","https://openalex.org/W563168251","https://openalex.org/W806202468","https://openalex.org/W882857864","https://openalex.org/W1423289960","https://openalex.org/W1589142931","https://openalex.org/W1603760080","https://openalex.org/W1642473876","https://openalex.org/W1800105588","https://openalex.org/W1826514372","https://openalex.org/W1873022895","https://openalex.org/W1877420198","https://openalex.org/W1966688239","https://openalex.org/W1971776182","https://openalex.org/W1978619075","https://openalex.org/W1979290264","https://openalex.org/W1979459287","https://openalex.org/W1984998634","https://openalex.org/W1999673860","https://openalex.org/W2007329840","https://openalex.org/W2022833100","https://openalex.org/W2026170849","https://openalex.org/W2037316754","https://openalex.org/W2040099364","https://openalex.org/W2046440499","https://openalex.org/W2113026003","https://openalex.org/W2128961774","https://openalex.org/W2133933998","https://openalex.org/W2158634186","https://openalex.org/W2168130919","https://openalex.org/W2232396709","https://openalex.org/W2263652042","https://openalex.org/W2267596186","https://openalex.org/W2408634597","https://openalex.org/W2410887582","https://openalex.org/W2471697135","https://openalex.org/W2537743619","https://openalex.org/W2811455890","https://openalex.org/W2914775513","https://openalex.org/W4233279977","https://openalex.org/W4237733380","https://openalex.org/W4245683071","https://openalex.org/W4247781538","https://openalex.org/W4285719527","https://openalex.org/W4388671105","https://openalex.org/W6611657660","https://openalex.org/W6615733128","https://openalex.org/W6635899362","https://openalex.org/W6639501518","https://openalex.org/W6752670219"],"related_works":["https://openalex.org/W4252527915","https://openalex.org/W2409287660","https://openalex.org/W2233357156","https://openalex.org/W1976489385","https://openalex.org/W4256317220","https://openalex.org/W1506225852","https://openalex.org/W2141414364","https://openalex.org/W2376514150","https://openalex.org/W1523265213","https://openalex.org/W2032193370"],"abstract_inverted_index":{"Continuous":[0,281],"practices,":[1,35],"i.e.,":[2],"continuous":[3,34,55,69,103,144,157,176,244,269,335,351],"integration,":[4],"delivery,":[5],"and":[6,19,24,42,48,53,74,78,83,108,136,153,163,167,182,189,195,199,217,221,229,234,259,278,292,299,312,323,353,359,366],"deployment,":[7],"are":[8,301,309],"the":[9,27,31,44,63,66,85,92,99,114,120,141,147,253,295,321,329,355],"software":[10,289],"development":[11],"industry":[12],"practices":[13,49,70,79,104,145,245,270,282,300],"that":[14,237,268],"enable":[15],"organizations":[16],"to":[17,39,71,275,349],"frequently":[18],"reliably":[20],"release":[21],"new":[22],"features":[23],"products.":[25],"With":[26],"increasing":[28,161],"interest":[29],"in":[30,80,146,156,170,184,192,246,328],"literature":[32,94],"on":[33,102,165],"it":[36],"is":[37],"important":[38,286],"systematically":[40,61],"review":[41,95],"synthesize":[43],"approaches,":[45,297],"tools,":[46,75,138,298],"challenges,":[47,307],"reported":[50,296],"for":[51,87,97,118,319,361],"adopting":[52],"implementing":[54],"practices.":[56],"This":[57,264],"paper":[58,265],"aimed":[59],"at":[60],"reviewing":[62,98,124],"state":[64],"of":[65,68,143,201,210,252,288,306,325,334,341,357,364],"art":[67],"classify":[72],"approaches":[73,135],"identify":[76,84],"challenges":[77,311],"this":[81],"regard,":[82],"gaps":[86],"future":[88,316],"research.":[89],"We":[90,112,131,204],"used":[91],"systematic":[93],"method":[96,117],"peer-reviewed":[100],"papers":[101,126,255],"published":[105],"between":[106],"2004":[107],"June":[109],"1,":[110],"2016.":[111],"applied":[113,274],"thematic":[115],"analysis":[116],"analyzing":[119],"data":[121],"extracted":[122],"from":[123],"69":[125],"selected":[127],"using":[128],"predefined":[129],"criteria.":[130],"have":[132,205,271,283],"identified":[133],"30":[134],"associated":[137],"which":[139,314],"facilitate":[140],"implementation":[142],"following":[148],"ways:":[149],"(1)":[150],"reducing":[151],"build":[152,166],"test":[154,168],"time":[155],"integration":[158],"(CI);":[159],"(2)":[160],"visibility":[162],"awareness":[164,220],"results":[169],"CI;":[171,185],"(3)":[172],"supporting":[173],"(semi-)":[174],"automated":[175],"testing;":[177],"(4)":[178],"detecting":[179],"violations,":[180],"flaws,":[181],"faults":[183],"(5)":[186],"addressing":[187,302,354],"security":[188],"scalability":[190],"issues":[191],"deployment":[193,202,369],"pipeline;":[194],"(6)":[196],"improving":[197,320],"dependability":[198],"reliability":[200],"process.":[203],"also":[206,266],"determined":[207],"a":[208,247,303,338],"list":[209],"critical":[211],"factors,":[212],"such":[213],"as":[214],"testing":[215],"(effort":[216],"time),":[218],"team":[219],"transparency,":[222],"good":[223],"design":[224],"principles,":[225],"customer,":[226],"highly":[227],"skilled":[228],"motivated":[230],"team,":[231],"application":[232],"domain,":[233],"appropriate":[235],"infrastructure":[236],"should":[238,345],"be":[239,346],"carefully":[240],"considered":[241],"when":[242],"introducing":[243],"given":[248],"organization.":[249],"The":[250],"majority":[251],"reviewed":[254],"were":[256],"validation":[257],"(34.7%)":[258],"evaluation":[260],"(36.2%)":[261],"research":[262,291,317],"types.":[263],"reveals":[267],"been":[272],"successfully":[273],"both":[276],"greenfield":[277],"maintenance":[279],"projects.":[280],"become":[284],"an":[285],"area":[287],"engineering":[290,362],"practice.":[293],"While":[294],"wide":[304],"range":[305],"there":[308],"several":[310],"gaps,":[313],"require":[315],"work":[318],"capturing":[322],"reporting":[324,331],"contextual":[326],"information":[327],"studies":[330],"different":[332],"aspects":[333],"practices;":[336,352],"gaining":[337],"deep":[339],"understanding":[340],"how":[342],"software-intensive":[343],"systems":[344],"(re-)":[347],"architected":[348],"support":[350],"lack":[356],"knowledge":[358],"tools":[360],"processes":[363],"designing":[365],"running":[367],"secure":[368],"pipelines.":[370]},"counts_by_year":[{"year":2026,"cited_by_count":34},{"year":2025,"cited_by_count":107},{"year":2024,"cited_by_count":89},{"year":2023,"cited_by_count":100},{"year":2022,"cited_by_count":90},{"year":2021,"cited_by_count":86},{"year":2020,"cited_by_count":49},{"year":2019,"cited_by_count":40},{"year":2018,"cited_by_count":34},{"year":2017,"cited_by_count":5}],"updated_date":"2026-05-22T09:01:20.584952","created_date":"2025-10-10T00:00:00"}