{"id":"https://openalex.org/W2347053969","doi":"https://doi.org/10.1145/2970276.2970358","title":"Usage, costs, and benefits of continuous integration in open-source projects","display_name":"Usage, costs, and benefits of continuous integration in open-source projects","publication_year":2016,"publication_date":"2016-08-25","ids":{"openalex":"https://openalex.org/W2347053969","doi":"https://doi.org/10.1145/2970276.2970358","mag":"2347053969"},"language":"en","primary_location":{"id":"doi:10.1145/2970276.2970358","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970276.2970358","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970358&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2970358&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088362471","display_name":"Michael Hilton","orcid":"https://orcid.org/0000-0001-9195-6902"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Michael Hilton","raw_affiliation_strings":["Oregon State University, USA"],"affiliations":[{"raw_affiliation_string":"Oregon State University, USA","institution_ids":["https://openalex.org/I131249849"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021297302","display_name":"Timothy Tunnell","orcid":null},"institutions":[{"id":"https://openalex.org/I2801919071","display_name":"University of Illinois System","ror":"https://ror.org/05e94g991","country_code":"US","type":"education","lineage":["https://openalex.org/I2801919071"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Timothy Tunnell","raw_affiliation_strings":["University of Illinois, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois, USA","institution_ids":["https://openalex.org/I2801919071"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100768454","display_name":"Kai Huang","orcid":"https://orcid.org/0009-0007-5772-7912"},"institutions":[{"id":"https://openalex.org/I2801919071","display_name":"University of Illinois System","ror":"https://ror.org/05e94g991","country_code":"US","type":"education","lineage":["https://openalex.org/I2801919071"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kai Huang","raw_affiliation_strings":["University of Illinois, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois, USA","institution_ids":["https://openalex.org/I2801919071"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081289583","display_name":"Darko Marinov","orcid":"https://orcid.org/0000-0001-5023-3492"},"institutions":[{"id":"https://openalex.org/I2801919071","display_name":"University of Illinois System","ror":"https://ror.org/05e94g991","country_code":"US","type":"education","lineage":["https://openalex.org/I2801919071"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Darko Marinov","raw_affiliation_strings":["University of Illinois, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois, USA","institution_ids":["https://openalex.org/I2801919071"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019404521","display_name":"Danny Dig","orcid":"https://orcid.org/0000-0001-5046-2017"},"institutions":[{"id":"https://openalex.org/I131249849","display_name":"Oregon State University","ror":"https://ror.org/00ysfqy60","country_code":"US","type":"education","lineage":["https://openalex.org/I131249849"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danny Dig","raw_affiliation_strings":["Oregon State University, USA"],"affiliations":[{"raw_affiliation_string":"Oregon State University, USA","institution_ids":["https://openalex.org/I131249849"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5088362471"],"corresponding_institution_ids":["https://openalex.org/I131249849"],"apc_list":null,"apc_paid":null,"fwci":62.333,"has_fulltext":true,"cited_by_count":284,"citation_normalized_percentile":{"value":0.99896851,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"426","last_page":"437"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5916515588760376},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.5628936886787415},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4988842010498047},{"id":"https://openalex.org/keywords/open-source-software","display_name":"Open source software","score":0.4633508324623108},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4431864321231842},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.41200608015060425},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.4108659327030182},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.37473374605178833},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3435501456260681},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.3232905864715576},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1868121325969696},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.11359390616416931}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5916515588760376},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.5628936886787415},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4988842010498047},{"id":"https://openalex.org/C2988343187","wikidata":"https://www.wikidata.org/wiki/Q1130645","display_name":"Open source software","level":3,"score":0.4633508324623108},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4431864321231842},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41200608015060425},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.4108659327030182},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.37473374605178833},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3435501456260681},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.3232905864715576},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1868121325969696},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.11359390616416931},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2970276.2970358","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970276.2970358","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970358&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2970276.2970358","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2970276.2970358","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2970358&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.5600000023841858,"id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G1394351670","display_name":null,"funder_award_id":"1439957","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1792573876","display_name":null,"funder_award_id":"CCF- 1421503, CCF-1439957, CCF-1553741","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2247958622","display_name":null,"funder_award_id":"CCF-1553741","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3975747979","display_name":null,"funder_award_id":"CCF-1439957","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4032422751","display_name":null,"funder_award_id":"CCF-1421503","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5186714734","display_name":"CAREER: Program Analysis and Transformations for Asynchrony","funder_award_id":"1553741","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5449669011","display_name":"SHF: Small: Revisiting Assumptions of Regression Testing","funder_award_id":"1421503","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2347053969.pdf","grobid_xml":"https://content.openalex.org/works/W2347053969.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W38316647","https://openalex.org/W1660704455","https://openalex.org/W1970122721","https://openalex.org/W1971972080","https://openalex.org/W1972386298","https://openalex.org/W1985896931","https://openalex.org/W1987174293","https://openalex.org/W1991292921","https://openalex.org/W1992786608","https://openalex.org/W1999128860","https://openalex.org/W1999673860","https://openalex.org/W2014515160","https://openalex.org/W2015762052","https://openalex.org/W2040915209","https://openalex.org/W2054811852","https://openalex.org/W2058879750","https://openalex.org/W2070544288","https://openalex.org/W2073433262","https://openalex.org/W2075519134","https://openalex.org/W2090386027","https://openalex.org/W2091561343","https://openalex.org/W2093927510","https://openalex.org/W2100987397","https://openalex.org/W2113848838","https://openalex.org/W2116553987","https://openalex.org/W2121016257","https://openalex.org/W2149185259","https://openalex.org/W2206426301","https://openalex.org/W2339061030","https://openalex.org/W2546542678","https://openalex.org/W6682087758"],"related_works":["https://openalex.org/W4376877853","https://openalex.org/W1493891899","https://openalex.org/W4250928611","https://openalex.org/W166480398","https://openalex.org/W1612808768","https://openalex.org/W167327709","https://openalex.org/W1977393088","https://openalex.org/W4387839566","https://openalex.org/W4210922265","https://openalex.org/W2288962794"],"abstract_inverted_index":{"Continuous":[0],"integration":[1],"(CI)":[2],"systems":[3,91],"automate":[4],"the":[5,31,80,112,142,157,172,181],"compilation,":[6],"building,":[7],"and":[8,43,48,60,145,194],"testing":[9],"of":[10,68,82,147,184],"software.":[11],"Despite":[12],"CI":[13,39,83,90,116,160,167,187],"rising":[14],"as":[15,176,178],"a":[16],"big":[17],"success":[18],"story":[19],"in":[20,41,84],"automated":[21],"software":[22],"engineering,":[23],"it":[24,192],"has":[25],"received":[26],"almost":[27],"no":[28],"attention":[29],"from":[30,99,111],"research":[32,199],"community.":[33],"For":[34],"example,":[35],"how":[36,103],"widely":[37,169],"is":[38,168],"used":[40,115],"practice,":[42],"what":[44],"are":[45],"some":[46],"costs":[47],"benefits":[49,146],"associated":[50],"with":[51],"CI?":[52],"Without":[53],"answering":[54],"such":[55],"questions,":[56],"developers,":[57],"tool":[58],"builders,":[59],"researchers":[61],"make":[62],"decisions":[63],"based":[64],"on":[65,200],"folklore":[66],"instead":[67],"data.":[69],"In":[70],"this":[71,133],"paper,":[72],"we":[73,94,107,128,135,152],"use":[74,105,122,126],"three":[75],"complementary":[76],"methods":[77],"to":[78,141,189,196],"study":[79],"usage":[81],"open-source":[85,97],"projects.":[86],"To":[87,101,118],"understand":[88,102,119],"which":[89],"developers":[92,104],"use,":[93],"analyzed":[95,108],"34,544":[96],"projects":[98,121,162,185],"GitHub.":[100],"CI,":[106,127],"1,529,291":[109],"builds":[110],"most":[113,173],"commonly":[114],"system.":[117],"why":[120],"or":[123],"do":[124],"not":[125],"surveyed":[129],"442":[130],"developers.":[131],"With":[132],"data,":[134],"answered":[136],"several":[137],"key":[138],"questions":[139],"related":[140],"usage,":[143],"costs,":[144],"CI.":[148,201],"Among":[149],"our":[150],"results,":[151],"show":[153],"evidence":[154],"that":[155,159,166,180],"supports":[156],"claim":[158],"helps":[161],"release":[163],"more":[164,198],"often,":[165],"adopted":[170],"by":[171],"popular":[174],"projects,":[175],"well":[177],"finding":[179],"overall":[182],"percentage":[183],"using":[186],"continues":[188],"grow,":[190],"making":[191],"important":[193],"timely":[195],"focus":[197]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":31},{"year":2023,"cited_by_count":29},{"year":2022,"cited_by_count":36},{"year":2021,"cited_by_count":39},{"year":2020,"cited_by_count":28},{"year":2019,"cited_by_count":36},{"year":2018,"cited_by_count":31},{"year":2017,"cited_by_count":20},{"year":2016,"cited_by_count":3}],"updated_date":"2026-03-18T14:38:29.013473","created_date":"2025-10-10T00:00:00"}