{"id":"https://openalex.org/W2804239311","doi":"https://doi.org/10.1109/tse.2018.2864633","title":"Analyzing Families of Experiments in SE: A Systematic Mapping Study","display_name":"Analyzing Families of Experiments in SE: A Systematic Mapping Study","publication_year":2018,"publication_date":"2018-08-09","ids":{"openalex":"https://openalex.org/W2804239311","doi":"https://doi.org/10.1109/tse.2018.2864633","mag":"2804239311"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2018.2864633","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2018.2864633","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1805.09009","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111559595","display_name":"Adri\u00e1n Santos","orcid":"https://orcid.org/0000-0002-8690-2309"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Adrian Santos","raw_affiliation_strings":["M3S (M-Group), ITEE University of Oulu, Oulu, Finland","[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]"],"raw_orcid":"https://orcid.org/0000-0002-8690-2309","affiliations":[{"raw_affiliation_string":"M3S (M-Group), ITEE University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]},{"raw_affiliation_string":"[M3S (M-Group), ITEE University of Oulu, Oulu, Finland]","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065451953","display_name":"Omar S. G\u00f3mez","orcid":"https://orcid.org/0000-0002-2951-3833"},"institutions":[{"id":"https://openalex.org/I4210137344","display_name":"Escuela Superior Polit\u00e9cnica del Chimborazo","ror":"https://ror.org/02zyw2q61","country_code":"EC","type":"education","lineage":["https://openalex.org/I4210137344"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Omar Gomez","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica de Chimborazo Riobamba, Chimborazo, Ecuador"],"raw_orcid":"https://orcid.org/0000-0002-2951-3833","affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica de Chimborazo Riobamba, Chimborazo, Ecuador","institution_ids":["https://openalex.org/I4210137344"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056117323","display_name":"Nat\u00e1lia Juristo","orcid":"https://orcid.org/0000-0002-2465-7141"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Natalia Juristo","raw_affiliation_strings":["Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Campus Montegancedo, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Campus Montegancedo, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.907,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.93082934,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":97},"biblio":{"volume":"46","issue":"5","first_page":"566","last_page":"583"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.6597385406494141},{"id":"https://openalex.org/keywords/data-aggregator","display_name":"Data aggregator","score":0.6394727230072021},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6174538135528564},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.6058122515678406},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5812206864356995},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5495051145553589},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5057826042175293},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4686049222946167},{"id":"https://openalex.org/keywords/aggregate-data","display_name":"Aggregate data","score":0.41978222131729126},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.11884400248527527},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10802176594734192},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.10667169094085693},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.07726669311523438}],"concepts":[{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.6597385406494141},{"id":"https://openalex.org/C82578977","wikidata":"https://www.wikidata.org/wiki/Q16773055","display_name":"Data aggregator","level":3,"score":0.6394727230072021},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6174538135528564},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.6058122515678406},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5812206864356995},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5495051145553589},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5057826042175293},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4686049222946167},{"id":"https://openalex.org/C2778058735","wikidata":"https://www.wikidata.org/wiki/Q4692253","display_name":"Aggregate data","level":2,"score":0.41978222131729126},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11884400248527527},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10802176594734192},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.10667169094085693},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.07726669311523438},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C24590314","wikidata":"https://www.wikidata.org/wiki/Q336038","display_name":"Wireless sensor network","level":2,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2018.2864633","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2018.2864633","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1805.09009","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.09009","pdf_url":"https://arxiv.org/pdf/1805.09009","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"mag:2804239311","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1805.09009.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1805.09009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1805.09009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1805.09009","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.09009","pdf_url":"https://arxiv.org/pdf/1805.09009","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4384623761","display_name":null,"funder_award_id":"TIN2014-60490-P","funder_id":"https://openalex.org/F4320322930","funder_display_name":"Ministerio de Ciencia e Innovaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320322930","display_name":"Ministerio de Ciencia e Innovaci\u00f3n","ror":"https://ror.org/034900433"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":127,"referenced_works":["https://openalex.org/W4214443","https://openalex.org/W24088030","https://openalex.org/W168533029","https://openalex.org/W608504492","https://openalex.org/W1470195681","https://openalex.org/W1490968191","https://openalex.org/W1539952819","https://openalex.org/W1541214726","https://openalex.org/W1566773348","https://openalex.org/W1567738769","https://openalex.org/W1598602811","https://openalex.org/W1841651683","https://openalex.org/W1909800943","https://openalex.org/W1948497384","https://openalex.org/W1964258385","https://openalex.org/W1975675278","https://openalex.org/W1977247477","https://openalex.org/W1981329136","https://openalex.org/W1983830972","https://openalex.org/W1986694300","https://openalex.org/W1988962172","https://openalex.org/W1989733111","https://openalex.org/W1990341655","https://openalex.org/W1998686495","https://openalex.org/W2000453772","https://openalex.org/W2003248283","https://openalex.org/W2003323761","https://openalex.org/W2006331688","https://openalex.org/W2007969951","https://openalex.org/W2008285675","https://openalex.org/W2009457315","https://openalex.org/W2010267787","https://openalex.org/W2016999376","https://openalex.org/W2018152980","https://openalex.org/W2020134952","https://openalex.org/W2025116371","https://openalex.org/W2037705678","https://openalex.org/W2037922987","https://openalex.org/W2043241676","https://openalex.org/W2044177369","https://openalex.org/W2044809867","https://openalex.org/W2050118439","https://openalex.org/W2055755944","https://openalex.org/W2055757491","https://openalex.org/W2056789386","https://openalex.org/W2058463394","https://openalex.org/W2059665864","https://openalex.org/W2061141931","https://openalex.org/W2064161267","https://openalex.org/W2077342771","https://openalex.org/W2077896602","https://openalex.org/W2077924088","https://openalex.org/W2078105379","https://openalex.org/W2078563187","https://openalex.org/W2081040380","https://openalex.org/W2083961577","https://openalex.org/W2084278640","https://openalex.org/W2084577255","https://openalex.org/W2087263043","https://openalex.org/W2092459872","https://openalex.org/W2094738656","https://openalex.org/W2095974519","https://openalex.org/W2099545896","https://openalex.org/W2104806923","https://openalex.org/W2105824687","https://openalex.org/W2110032017","https://openalex.org/W2111199933","https://openalex.org/W2113178124","https://openalex.org/W2113533445","https://openalex.org/W2115409003","https://openalex.org/W2120865520","https://openalex.org/W2122010200","https://openalex.org/W2129574978","https://openalex.org/W2131154789","https://openalex.org/W2138428785","https://openalex.org/W2140017498","https://openalex.org/W2140984588","https://openalex.org/W2144396040","https://openalex.org/W2144683894","https://openalex.org/W2146773231","https://openalex.org/W2147527890","https://openalex.org/W2148540129","https://openalex.org/W2149827333","https://openalex.org/W2150683078","https://openalex.org/W2152425345","https://openalex.org/W2155988679","https://openalex.org/W2158524254","https://openalex.org/W2161222712","https://openalex.org/W2162411541","https://openalex.org/W2164425715","https://openalex.org/W2167042818","https://openalex.org/W2167926541","https://openalex.org/W2168282376","https://openalex.org/W2168894761","https://openalex.org/W2168991419","https://openalex.org/W2205176590","https://openalex.org/W2249595199","https://openalex.org/W2267952752","https://openalex.org/W2293815892","https://openalex.org/W2339971975","https://openalex.org/W2441124914","https://openalex.org/W2488902449","https://openalex.org/W2500762131","https://openalex.org/W2509138770","https://openalex.org/W2510550031","https://openalex.org/W2584675329","https://openalex.org/W2589415147","https://openalex.org/W2617805303","https://openalex.org/W2764311721","https://openalex.org/W2905399339","https://openalex.org/W3013019166","https://openalex.org/W4235011715","https://openalex.org/W4238354999","https://openalex.org/W4239527626","https://openalex.org/W4247781538","https://openalex.org/W4291213652","https://openalex.org/W4300869678","https://openalex.org/W6606868457","https://openalex.org/W6632399127","https://openalex.org/W6642353749","https://openalex.org/W6651723475","https://openalex.org/W6682213787","https://openalex.org/W6684453849","https://openalex.org/W6684456340","https://openalex.org/W6687893060","https://openalex.org/W6713163372","https://openalex.org/W6745148249"],"related_works":["https://openalex.org/W2963392119","https://openalex.org/W3196894413","https://openalex.org/W2991315158","https://openalex.org/W2042338565","https://openalex.org/W2914868215","https://openalex.org/W2963570409","https://openalex.org/W1928031482","https://openalex.org/W2041237963","https://openalex.org/W2158879251","https://openalex.org/W2590418255","https://openalex.org/W2978236720","https://openalex.org/W2739939176","https://openalex.org/W2894935611","https://openalex.org/W3112354172","https://openalex.org/W2059265955","https://openalex.org/W2891521002","https://openalex.org/W1982848337","https://openalex.org/W2044045982","https://openalex.org/W2965579526","https://openalex.org/W2095166192"],"abstract_inverted_index":{"Context:":[0],"Families":[1,173],"of":[2,6,53,56,84,94,117,121,142,159,174,187,205,220],"experiments":[3,7,85,118,175],"(i.e.,":[4],"groups":[5],"with":[8,125,179],"the":[9,14,40,54,77,90,185,201],"same":[10],"goal)":[11],"are":[12,176],"on":[13],"rise":[15],"in":[16,49,86,119],"Software":[17],"Engineering":[18],"(SE).":[19],"Selecting":[20],"unsuitable":[21,180],"aggregation":[22,59,78,96,133,166,181],"techniques":[23,41,60,79,134,167,182],"to":[24,31,43,61,75,81,99,112,127,138,164,184,199,213,216],"analyze":[25,82,113,139,217],"families":[26,48,83,116,141,219],"may":[27],"undermine":[28],"their":[29],"potential":[30],"provide":[32,109],"in-depth":[33],"insights":[34],"from":[35],"experiments'":[36,45],"results.":[37,207],"Objectives:":[38],"Identifying":[39],"used":[42,80,137,163],"aggregate":[44],"results":[46],"within":[47,65,171],"SE.":[50,87],"Raising":[51],"awareness":[52],"importance":[55],"applying":[57],"suitable":[58,215],"reach":[62],"reliable":[63],"conclusions":[64],"families.":[66,172],"Method:":[67],"We":[68,88,108],"conduct":[69],"a":[70],"systematic":[71],"mapping":[72],"study":[73],"(SMS)":[74],"identify":[76],"outline":[89],"advantages":[91],"and":[92,106,114,157,203,209],"disadvantages":[93],"each":[95],"technique":[97],"according":[98,183],"mature":[100,188],"experimental":[101,189],"disciplines":[102],"such":[103],"as":[104],"medicine":[105],"pharmacology.":[107],"preliminary":[110],"recommendations":[111],"report":[115],"view":[120],"families'":[122],"common":[123],"limitations":[124],"regard":[126],"joint":[128,206],"data":[129],"analysis.":[130],"Results:":[131],"Several":[132],"have":[135],"been":[136],"SE":[140,218],"experiments,":[143],"including":[144],"Narrative":[145],"synthesis,":[146],"Aggregated":[147],"Data":[148,152,192],"(AD),":[149],"Individual":[150],"Participant":[151],"(IPD)":[153],"mega-trial":[154],"or":[155],"stratified,":[156],"Aggregation":[158],"p-values.":[160],"The":[161],"rationale":[162],"select":[165],"is":[168],"rarely":[169],"discussed":[170],"commonly":[177],"analyzed":[178],"literature":[186],"disciplines.":[190],"Conclusion:":[191],"analysis'":[193],"reporting":[194],"practices":[195],"should":[196],"be":[197,214],"improved":[198],"increase":[200],"reliability":[202],"transparency":[204],"AD":[208],"IPD":[210],"stratified":[211],"appear":[212],"experiments.":[221]},"counts_by_year":[{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}