{"id":"https://openalex.org/W2969699495","doi":"https://doi.org/10.1109/tse.2019.2935720","title":"A Procedure and Guidelines for Analyzing Groups of Software Engineering Replications","display_name":"A Procedure and Guidelines for Analyzing Groups of Software Engineering Replications","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2969699495","doi":"https://doi.org/10.1109/tse.2019.2935720","mag":"2969699495"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2019.2935720","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2019.2935720","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111559595","display_name":"Adri\u00e1n Santos","orcid":null},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":true,"raw_author_name":"Adrian Santos","raw_affiliation_strings":["M3S (M-Group), ITEE University of Oulu, Oulu, Finland","[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]"],"affiliations":[{"raw_affiliation_string":"M3S (M-Group), ITEE University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]},{"raw_affiliation_string":"[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076034572","display_name":"Sira Vegas","orcid":"https://orcid.org/0000-0001-8535-9386"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Sira Vegas","raw_affiliation_strings":["Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain","Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain#TAB#"],"affiliations":[{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]},{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain#TAB#","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064264297","display_name":"Markku Oivo","orcid":"https://orcid.org/0000-0002-1698-2323"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Markku Oivo","raw_affiliation_strings":["M3S (M-Group), ITEE University of Oulu, Oulu, Finland","[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]"],"affiliations":[{"raw_affiliation_string":"M3S (M-Group), ITEE University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]},{"raw_affiliation_string":"[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056117323","display_name":"Nat\u00e1lia Juristo","orcid":"https://orcid.org/0000-0002-2465-7141"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Natalia Juristo","raw_affiliation_strings":["Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain","Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain#TAB#"],"affiliations":[{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]},{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Madrid, Spain#TAB#","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5111559595"],"corresponding_institution_ids":["https://openalex.org/I98381234"],"apc_list":null,"apc_paid":null,"fwci":4.7481,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":{"value":0.95413798,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.9896000027656555,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/replication","display_name":"Replication (statistics)","score":0.8217625617980957},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7059333324432373},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5274173021316528},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4874587059020996},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.48314231634140015},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.47454890608787537},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.4672655463218689},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.4468485116958618},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.354106068611145},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.21037790179252625},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14904603362083435},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08238813281059265}],"concepts":[{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.8217625617980957},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7059333324432373},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5274173021316528},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4874587059020996},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.48314231634140015},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.47454890608787537},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.4672655463218689},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.4468485116958618},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.354106068611145},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.21037790179252625},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14904603362083435},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08238813281059265},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tse.2019.2935720","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2019.2935720","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:oulu.fi:nbnfi-fe2019120946189","is_oa":false,"landing_page_url":"http://urn.fi/urn:nbn:fi-fe2019120946189","pdf_url":null,"source":{"id":"https://openalex.org/S4306400284","display_name":"University of Oulu Repository (University of Oulu)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98381234","host_organization_name":"University of Oulu","host_organization_lineage":["https://openalex.org/I98381234"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5956360995","display_name":null,"funder_award_id":"PGC2018- 097265-B-I00","funder_id":"https://openalex.org/F4320315062","funder_display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades"}],"funders":[{"id":"https://openalex.org/F4320315062","display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":144,"referenced_works":["https://openalex.org/W24088030","https://openalex.org/W563289791","https://openalex.org/W608504492","https://openalex.org/W1447279895","https://openalex.org/W1489838709","https://openalex.org/W1508276673","https://openalex.org/W1511267087","https://openalex.org/W1518897246","https://openalex.org/W1522872137","https://openalex.org/W1534431701","https://openalex.org/W1598602811","https://openalex.org/W1604979160","https://openalex.org/W1730782591","https://openalex.org/W1841651683","https://openalex.org/W1885214473","https://openalex.org/W1900885688","https://openalex.org/W1911264425","https://openalex.org/W1923600032","https://openalex.org/W1948497384","https://openalex.org/W1974992958","https://openalex.org/W1985969598","https://openalex.org/W1990341655","https://openalex.org/W1991369112","https://openalex.org/W2000453772","https://openalex.org/W2003323761","https://openalex.org/W2007969951","https://openalex.org/W2008285675","https://openalex.org/W2009432182","https://openalex.org/W2009457315","https://openalex.org/W2009580338","https://openalex.org/W2013222969","https://openalex.org/W2015709452","https://openalex.org/W2015804289","https://openalex.org/W2016832451","https://openalex.org/W2018152980","https://openalex.org/W2021466689","https://openalex.org/W2029266105","https://openalex.org/W2030360178","https://openalex.org/W2034109986","https://openalex.org/W2037922987","https://openalex.org/W2044809867","https://openalex.org/W2049192772","https://openalex.org/W2049406759","https://openalex.org/W2050118439","https://openalex.org/W2052335770","https://openalex.org/W2055755944","https://openalex.org/W2055757491","https://openalex.org/W2056789386","https://openalex.org/W2056825323","https://openalex.org/W2058806401","https://openalex.org/W2059665864","https://openalex.org/W2064161267","https://openalex.org/W2066707382","https://openalex.org/W2070230859","https://openalex.org/W2075711546","https://openalex.org/W2076029586","https://openalex.org/W2077342771","https://openalex.org/W2084007633","https://openalex.org/W2084577255","https://openalex.org/W2091979577","https://openalex.org/W2092459872","https://openalex.org/W2094315418","https://openalex.org/W2094738656","https://openalex.org/W2095437242","https://openalex.org/W2095974519","https://openalex.org/W2103458796","https://openalex.org/W2108696783","https://openalex.org/W2110032017","https://openalex.org/W2111199933","https://openalex.org/W2111680408","https://openalex.org/W2112146495","https://openalex.org/W2113178124","https://openalex.org/W2113533445","https://openalex.org/W2114269784","https://openalex.org/W2115409003","https://openalex.org/W2118248224","https://openalex.org/W2120865520","https://openalex.org/W2128313432","https://openalex.org/W2129005287","https://openalex.org/W2131154789","https://openalex.org/W2133940850","https://openalex.org/W2136618123","https://openalex.org/W2138432709","https://openalex.org/W2140017498","https://openalex.org/W2141438777","https://openalex.org/W2141767681","https://openalex.org/W2144683894","https://openalex.org/W2146773231","https://openalex.org/W2148540129","https://openalex.org/W2150639273","https://openalex.org/W2150683078","https://openalex.org/W2156267802","https://openalex.org/W2159072822","https://openalex.org/W2160489028","https://openalex.org/W2161222712","https://openalex.org/W2161465471","https://openalex.org/W2167926541","https://openalex.org/W2168282376","https://openalex.org/W2168894761","https://openalex.org/W2168991419","https://openalex.org/W2249595199","https://openalex.org/W2293040502","https://openalex.org/W2295320765","https://openalex.org/W2302202015","https://openalex.org/W2339971975","https://openalex.org/W2424693453","https://openalex.org/W2489534557","https://openalex.org/W2500762131","https://openalex.org/W2533466707","https://openalex.org/W2582364179","https://openalex.org/W2593708391","https://openalex.org/W2612764117","https://openalex.org/W2617805303","https://openalex.org/W2622958941","https://openalex.org/W2762364541","https://openalex.org/W2778005203","https://openalex.org/W2790894500","https://openalex.org/W2791681873","https://openalex.org/W2801901298","https://openalex.org/W2805760149","https://openalex.org/W2883684507","https://openalex.org/W2884313467","https://openalex.org/W2890373521","https://openalex.org/W2890801208","https://openalex.org/W2963392119","https://openalex.org/W3100196413","https://openalex.org/W4214744654","https://openalex.org/W4230983304","https://openalex.org/W4231259605","https://openalex.org/W4235011715","https://openalex.org/W4235249396","https://openalex.org/W4239350751","https://openalex.org/W4239527626","https://openalex.org/W4242721106","https://openalex.org/W4247781538","https://openalex.org/W4247949085","https://openalex.org/W4249888275","https://openalex.org/W4291213652","https://openalex.org/W4298188426","https://openalex.org/W4301655662","https://openalex.org/W6683065801","https://openalex.org/W6684456340","https://openalex.org/W6697204939","https://openalex.org/W6734353067"],"related_works":["https://openalex.org/W4205713785","https://openalex.org/W3016766501","https://openalex.org/W2901033488","https://openalex.org/W2031325922","https://openalex.org/W3086458689","https://openalex.org/W4287666211","https://openalex.org/W2898161671","https://openalex.org/W2249859249","https://openalex.org/W1594042954","https://openalex.org/W2354111411"],"abstract_inverted_index":{"Context:":[0],"Researchers":[1],"from":[2,55],"different":[3],"groups":[4,11,20,31,48,82,112,126,146,206,217],"and":[5,87,95,116,199,231],"institutions":[6],"are":[7,26],"collaborating":[8],"on":[9],"building":[10],"of":[12,16,21,32,36,47,49,66,81,83,99,111,113,127,137,145,147,160,182,194,207,218,233],"experiments":[13],"by":[14],"means":[15],"replication":[17,41,74,183],"(i.e.,":[18],"conducting":[19],"replications).":[22],"Disparate":[23],"aggregation":[24,181,212],"techniques":[25,38,213],"being":[27],"applied":[28],"to":[29,39,51,69,106,124,142,156,163,204,215],"analyze":[30,125,205,216],"replications.":[33,149,209],"The":[34,185,211,236],"application":[35],"unsuitable":[37],"aggregate":[40,70],"results":[42],"may":[43],"undermine":[44],"the":[45,79,102,107,117,143,152,169,173,180,192,229],"potential":[46],"replications":[50,84,115,162,219],"provide":[52],"in-depth":[53],"insights":[54],"experiment":[56],"results.":[57,75,184,235],"Objectives:":[58],"Provide":[59],"an":[60,131],"analysis":[61,110,132,144,154,186],"procedure":[62,133,155,187],"with":[63,104,134],"a":[64,135,157],"set":[65,136],"embedded":[67,138],"guidelines":[68,118,139,238],"software":[71],"engineering":[72],"(SE)":[73],"Method:":[76],"We":[77,150],"compare":[78],"characteristics":[80],"for":[85],"SE":[86,114,148,161,208],"other":[88],"mature":[89,121],"experimental":[90,122],"disciplines":[91,123],"such":[92],"as":[93],"medicine":[94],"pharmacology.":[96],"In":[97],"view":[98],"their":[100],"differences,":[101],"limitations":[103],"regard":[105],"joint":[108,234],"data":[109,175,198,201],"provided":[119],"in":[120,202,223],"replications,":[128],"we":[129,189],"build":[130],"specifically":[140],"tailored":[141],"apply":[151],"proposed":[153,237],"representative":[158],"group":[159],"illustrate":[164],"its":[165],"use.":[166],"Results:":[167],"All":[168],"information":[170],"contained":[171],"within":[172],"raw":[174],"should":[176,220,239],"be":[177,221],"leveraged":[178],"during":[179],"that":[188],"propose":[190],"encourages":[191],"use":[193],"stratified":[195],"individual":[196],"participant":[197],"aggregated":[200],"tandem":[203],"Conclusion:":[210],"used":[214],"justified":[222],"research":[224],"articles.":[225],"This":[226],"will":[227],"increase":[228],"reliability":[230],"transparency":[232],"ease":[240],"this":[241],"endeavor.":[242]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":7},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}