{"id":"https://openalex.org/W2090625011","doi":"https://doi.org/10.1007/s10664-011-9181-9","title":"On the reproducibility of empirical software engineering studies based on data retrieved from development repositories","display_name":"On the reproducibility of empirical software engineering studies based on data retrieved from development repositories","publication_year":2011,"publication_date":"2011-10-17","ids":{"openalex":"https://openalex.org/W2090625011","doi":"https://doi.org/10.1007/s10664-011-9181-9","mag":"2090625011"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-011-9181-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-011-9181-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-011-9181-9.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-011-9181-9.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5066580987","display_name":"Jes\u00fas M. Gonz\u00e1lez-Barahona","orcid":"https://orcid.org/0000-0001-9682-460X"},"institutions":[{"id":"https://openalex.org/I182083151","display_name":"Universidad Rey Juan Carlos","ror":"https://ror.org/01v5cv687","country_code":"ES","type":"education","lineage":["https://openalex.org/I182083151"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Jes\u00fas M. Gonz\u00e1lez-Barahona","raw_affiliation_strings":["Universidad Rey Juan Carlos, Mostoles, Spain","Universidad Rey Juan Carlos, Mostoles, Spain#TAB#"],"affiliations":[{"raw_affiliation_string":"Universidad Rey Juan Carlos, Mostoles, Spain","institution_ids":["https://openalex.org/I182083151"]},{"raw_affiliation_string":"Universidad Rey Juan Carlos, Mostoles, Spain#TAB#","institution_ids":["https://openalex.org/I182083151"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5061131972","display_name":"Greg\u00f3rio Robles","orcid":"https://orcid.org/0000-0002-1442-6761"},"institutions":[{"id":"https://openalex.org/I182083151","display_name":"Universidad Rey Juan Carlos","ror":"https://ror.org/01v5cv687","country_code":"ES","type":"education","lineage":["https://openalex.org/I182083151"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Gregorio Robles","raw_affiliation_strings":["Universidad Rey Juan Carlos, Fuenlabrada, Spain","[Universidad Rey Juan Carlos, Fuenlabrada, Spain]"],"affiliations":[{"raw_affiliation_string":"Universidad Rey Juan Carlos, Fuenlabrada, Spain","institution_ids":["https://openalex.org/I182083151"]},{"raw_affiliation_string":"[Universidad Rey Juan Carlos, Fuenlabrada, Spain]","institution_ids":["https://openalex.org/I182083151"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5066580987"],"corresponding_institution_ids":["https://openalex.org/I182083151"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":22.169,"has_fulltext":true,"cited_by_count":105,"citation_normalized_percentile":{"value":0.99235974,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"17","issue":"1-2","first_page":"75","last_page":"89"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.7985222339630127},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7267804145812988},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5838314294815063},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5039140582084656},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.49312159419059753},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.47441455721855164},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.40153563022613525},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.0807180404663086},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07432112097740173}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.7985222339630127},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7267804145812988},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5838314294815063},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5039140582084656},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.49312159419059753},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.47441455721855164},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40153563022613525},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0807180404663086},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07432112097740173},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s10664-011-9181-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-011-9181-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-011-9181-9.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:eciencia.urjc.es:10115/11302","is_oa":true,"landing_page_url":"http://hdl.handle.net/10115/11302","pdf_url":null,"source":{"id":"https://openalex.org/S4377196484","display_name":"BURJC Digital (King Juan Carlos University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I182083151","host_organization_name":"Universidad Rey Juan Carlos","host_organization_lineage":["https://openalex.org/I182083151"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1007/s10664-011-9181-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-011-9181-9","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-011-9181-9.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6488784250","display_name":"Active support and reaL-time coordination based on Event pRocessing in open source software developmenT","funder_award_id":"258098","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2090625011.pdf","grobid_xml":"https://content.openalex.org/works/W2090625011.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W168970045","https://openalex.org/W210239574","https://openalex.org/W2025865172","https://openalex.org/W2030620341","https://openalex.org/W2034158106","https://openalex.org/W2050136587","https://openalex.org/W2052942032","https://openalex.org/W2080505681","https://openalex.org/W2097111276","https://openalex.org/W2104279568","https://openalex.org/W2111861834","https://openalex.org/W2113533445","https://openalex.org/W2119731669","https://openalex.org/W2122175346","https://openalex.org/W2125733548","https://openalex.org/W2130072301","https://openalex.org/W2131883862","https://openalex.org/W2140017498","https://openalex.org/W2143453401","https://openalex.org/W2152759400","https://openalex.org/W2158685633","https://openalex.org/W2167716326","https://openalex.org/W2167926541","https://openalex.org/W2168874709","https://openalex.org/W2321915840","https://openalex.org/W4231430104"],"related_works":["https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W2140154150","https://openalex.org/W638612878","https://openalex.org/W4324138256","https://openalex.org/W2083330954"],"abstract_inverted_index":{"Among":[0],"empirical":[1],"software":[2],"engineering":[3],"studies,":[4],"those":[5,15],"based":[6],"on":[7],"data":[8],"retrieved":[9],"from":[10,38],"development":[11],"repositories":[12],"(such":[13],"as":[14,138],"of":[16,57,74,82,86,92,100,107,131,154,200,212],"source":[17],"code":[18],"management,":[19],"issue":[20],"tracking":[21],"or":[22,79,179],"communication":[23],"systems)":[24],"are":[25],"specially":[26],"suitable":[27],"for":[28,127,163,205],"reproduction.":[29],"However":[30],"their":[31],"reproducibility":[32,56,99,130,148],"status":[33],"can":[34,50,66,194,202],"vary":[35],"a":[36,58,93,101,125,139,169,206],"lot,":[37],"easy":[39],"to":[40,43,53,69,96,104,116,141,143,166,191],"almost":[41],"impossible":[42],"reproduce.":[44],"This":[45],"paper":[46],"explores":[47],"which":[48,198],"elements":[49,187],"be":[51,67,117,167,195,203],"considered":[52],"characterize":[54],"the":[55,72,83,90,98,105,108,113,129,155,210],"study":[59],"in":[60,149,157],"this":[61,87,123,150],"area,":[62],"and":[63,112,136,188,215],"how":[64],"they":[65,77],"analyzed":[68],"better":[70,196],"understand":[71],"type":[73],"reproduction":[75,177,201],"studies":[76,132],"enable":[78],"obstruct.":[80],"One":[81],"main":[84],"results":[85],"exploration":[88],"is":[89,133],"need":[91],"systematic":[94,170],"approach":[95],"asses":[97],"study,":[102,208],"due":[103],"complexity":[106],"processes":[109],"usually":[110],"involved,":[111],"many":[114],"details":[115],"taken":[118],"into":[119],"account.":[120],"To":[121],"address":[122],"need,":[124],"methodology":[126,156],"assessing":[128],"also":[134,182],"presented":[135],"discussed,":[137],"tool":[140],"help":[142],"raise":[144],"awareness":[145],"about":[146],"research":[147],"field.":[151],"The":[152],"application":[153],"practice":[158],"has":[159],"shown":[160],"how,":[161,184],"even":[162],"papers":[164],"aimed":[165],"reproducible,":[168],"analysis":[171],"raises":[172],"important":[173],"aspects":[174],"that":[175],"render":[176],"difficult":[178],"impossible.":[180],"We":[181],"show":[183],"by":[185],"identifying":[186],"attributes":[189],"related":[190],"reproducibility,":[192],"it":[193,217],"understood":[197],"kind":[199],"done":[204],"specific":[207],"given":[209],"description":[211],"datasets,":[213],"methodologies":[214],"parameters":[216],"uses.":[218]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":6},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":9},{"year":2017,"cited_by_count":6},{"year":2016,"cited_by_count":7},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":8},{"year":2013,"cited_by_count":12},{"year":2012,"cited_by_count":9}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2016-06-24T00:00:00"}