{"id":"https://openalex.org/W2804239311","doi":"https://doi.org/10.1109/tse.2018.2864633","title":"Analyzing Families of Experiments in SE: A Systematic Mapping Study","display_name":"Analyzing Families of Experiments in SE: A Systematic Mapping Study","publication_year":2018,"publication_date":"2018-08-09","ids":{"openalex":"https://openalex.org/W2804239311","doi":"https://doi.org/10.1109/tse.2018.2864633","mag":"2804239311"},"language":"en","primary_location":{"id":"doi:10.1109/tse.2018.2864633","doi":"https://doi.org/10.1109/tse.2018.2864633","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2018.2864633","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1805.09009","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5111559595","display_name":"Adri\u00e1n Santos","orcid":"https://orcid.org/0000-0002-8690-2309","observed_orcids":["https://orcid.org/0000-0002-8690-2309"]},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Adrian Santos","raw_affiliation_strings":["M3S (M-Group), ITEE University of Oulu, Oulu, Finland"],"raw_orcid":"https://orcid.org/0000-0002-8690-2309","affiliations":[{"raw_affiliation_string":"M3S (M-Group), ITEE University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065451953","display_name":"Omar S. G\u00f3mez","orcid":"https://orcid.org/0000-0002-2951-3833","observed_orcids":["https://orcid.org/0000-0002-2951-3833"]},"institutions":[{"id":"https://openalex.org/I4210137344","display_name":"Escuela Superior Polit\u00e9cnica de Chimborazo","ror":"https://ror.org/02zyw2q61","country_code":"EC","type":"education","lineage":["https://openalex.org/I4210137344"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Omar Gomez","raw_affiliation_strings":["Escuela Superior Polit\u00e9cnica de Chimborazo Riobamba, Chimborazo, Ecuador"],"raw_orcid":"https://orcid.org/0000-0002-2951-3833","affiliations":[{"raw_affiliation_string":"Escuela Superior Polit\u00e9cnica de Chimborazo Riobamba, Chimborazo, Ecuador","institution_ids":["https://openalex.org/I4210137344"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056117323","display_name":"Nat\u00e1lia Juristo","orcid":"https://orcid.org/0000-0002-2465-7141","observed_orcids":["https://orcid.org/0000-0002-2465-7141"]},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Natalia Juristo","raw_affiliation_strings":["Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Campus Montegancedo, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Campus Montegancedo, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2645,"currency":"USD","value_usd":2645},"apc_paid":null,"fwci":0.777,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.80063291,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":98},"biblio":{"volume":"46","issue":"5","first_page":"566","last_page":"583"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.6011000275611877,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.6011000275611877,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.3677000105381012,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.004900000058114529,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/aggregation-methods","display_name":"aggregation methods","score":0.7630000114440918},{"id":"https://openalex.org/keywords/meta-analysis","display_name":"meta-analysis","score":0.6370000243186951},{"id":"https://openalex.org/keywords/narrative-synthesis","display_name":"narrative synthesis","score":0.7409999966621399},{"id":"https://openalex.org/keywords/aggregate-data","display_name":"aggregated data","score":0.8970000147819519},{"id":"https://openalex.org/keywords/individual-patient-data","display_name":"individual patient data","score":0.9810000061988831},{"id":"https://openalex.org/keywords/reporting-practices","display_name":"reporting practices","score":0.6119999885559082},{"id":"https://openalex.org/keywords/software-engineering-experiments","display_name":"software engineering experiments","score":0.7720000147819519}],"concepts":[{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.6597385406494141},{"id":"https://openalex.org/C82578977","wikidata":"https://www.wikidata.org/wiki/Q16773055","display_name":"Data aggregator","level":3,"score":0.6394727230072021},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6174538135528564},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.6058122515678406},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5812206864356995},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5495051145553589},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5057826042175293},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4686049222946167},{"id":"https://openalex.org/C2778058735","wikidata":"https://www.wikidata.org/wiki/Q4692253","display_name":"Aggregate data","level":2,"score":0.41978222131729126},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11884400248527527},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10802176594734192},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.10667169094085693},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.07726669311523438},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C24590314","wikidata":"https://www.wikidata.org/wiki/Q336038","display_name":"Wireless sensor network","level":2,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tse.2018.2864633","doi":"https://doi.org/10.1109/tse.2018.2864633","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tse.2018.2864633","pdf_url":null,"source":{"id":"https://openalex.org/S8351582","display_name":"IEEE Transactions on Software Engineering","issn_l":"0098-5589","issn":["0098-5589","1939-3520","2326-3881"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-a","cwts-core","dongbi-a","fnege-3","jufo-3","norway-2","poland-140","russia-white-list-1"],"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1805.09009","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.09009","pdf_url":"https://arxiv.org/pdf/1805.09009","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2804239311","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1805.09009.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1805.09009","doi":"https://doi.org/10.48550/arxiv.1805.09009","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1805.09009","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1805.09009","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/1805.09009","pdf_url":"https://arxiv.org/pdf/1805.09009","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[{"id":"https://openalex.org/study-designs/systematic-review","display_name":"Systematic Review"}],"awards":[{"id":"https://openalex.org/G4384623761","display_name":null,"funder_award_id":"TIN2014-60490-P","funder_id":"https://openalex.org/F4320322930","funder_display_name":"Ministerio de Ciencia e Innovaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320322930","display_name":"Ministerio de Ciencia e Innovaci\u00f3n","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2804239311.pdf","grobid_xml":"https://content.openalex.org/works/W2804239311.grobid-xml"},"referenced_works_count":121,"referenced_works":["https://openalex.org/W4214443","https://openalex.org/W24088030","https://openalex.org/W168533029","https://openalex.org/W608504492","https://openalex.org/W1470195681","https://openalex.org/W1490968191","https://openalex.org/W1539952819","https://openalex.org/W1541214726","https://openalex.org/W1566773348","https://openalex.org/W1567738769","https://openalex.org/W1598602811","https://openalex.org/W1841651683","https://openalex.org/W1909800943","https://openalex.org/W1948497384","https://openalex.org/W1964258385","https://openalex.org/W1967931023","https://openalex.org/W1975675278","https://openalex.org/W1977247477","https://openalex.org/W1981329136","https://openalex.org/W1983830972","https://openalex.org/W1986694300","https://openalex.org/W1988962172","https://openalex.org/W1989733111","https://openalex.org/W1990341655","https://openalex.org/W1998686495","https://openalex.org/W2000453772","https://openalex.org/W2003248283","https://openalex.org/W2003323761","https://openalex.org/W2006331688","https://openalex.org/W2007969951","https://openalex.org/W2008285675","https://openalex.org/W2009457315","https://openalex.org/W2010267787","https://openalex.org/W2016999376","https://openalex.org/W2018152980","https://openalex.org/W2020134952","https://openalex.org/W2025116371","https://openalex.org/W2037705678","https://openalex.org/W2037922987","https://openalex.org/W2043241676","https://openalex.org/W2044177369","https://openalex.org/W2044809867","https://openalex.org/W2050118439","https://openalex.org/W2055755944","https://openalex.org/W2055757491","https://openalex.org/W2056789386","https://openalex.org/W2058463394","https://openalex.org/W2059665864","https://openalex.org/W2061141931","https://openalex.org/W2064161267","https://openalex.org/W2077342771","https://openalex.org/W2077896602","https://openalex.org/W2077924088","https://openalex.org/W2078105379","https://openalex.org/W2078563187","https://openalex.org/W2081040380","https://openalex.org/W2083961577","https://openalex.org/W2084278640","https://openalex.org/W2084577255","https://openalex.org/W2087263043","https://openalex.org/W2092459872","https://openalex.org/W2094738656","https://openalex.org/W2095974519","https://openalex.org/W2099545896","https://openalex.org/W2104806923","https://openalex.org/W2105824687","https://openalex.org/W2110032017","https://openalex.org/W2111199933","https://openalex.org/W2113178124","https://openalex.org/W2113533445","https://openalex.org/W2115409003","https://openalex.org/W2120865520","https://openalex.org/W2122010200","https://openalex.org/W2129574978","https://openalex.org/W2131154789","https://openalex.org/W2138428785","https://openalex.org/W2140017498","https://openalex.org/W2140984588","https://openalex.org/W2144396040","https://openalex.org/W2144683894","https://openalex.org/W2146773231","https://openalex.org/W2147527890","https://openalex.org/W2148540129","https://openalex.org/W2149827333","https://openalex.org/W2150683078","https://openalex.org/W2152425345","https://openalex.org/W2155988679","https://openalex.org/W2158524254","https://openalex.org/W2161222712","https://openalex.org/W2162411541","https://openalex.org/W2164425715","https://openalex.org/W2167042818","https://openalex.org/W2167926541","https://openalex.org/W2168282376","https://openalex.org/W2168894761","https://openalex.org/W2168991419","https://openalex.org/W2205176590","https://openalex.org/W2207729848","https://openalex.org/W2249595199","https://openalex.org/W2267952752","https://openalex.org/W2293815892","https://openalex.org/W2339971975","https://openalex.org/W2401001747","https://openalex.org/W2441124914","https://openalex.org/W2488902449","https://openalex.org/W2500762131","https://openalex.org/W2509138770","https://openalex.org/W2510550031","https://openalex.org/W2584675329","https://openalex.org/W2589415147","https://openalex.org/W2617805303","https://openalex.org/W2764311721","https://openalex.org/W2905399339","https://openalex.org/W3013019166","https://openalex.org/W4235011715","https://openalex.org/W4238354999","https://openalex.org/W4247781538","https://openalex.org/W4291213652","https://openalex.org/W4300869678","https://openalex.org/W6682213787","https://openalex.org/W6684453849"],"related_works":["https://openalex.org/W6991100106","https://openalex.org/W2969699495","https://openalex.org/W6995988977","https://openalex.org/W3108461549","https://openalex.org/W7005875415","https://openalex.org/W3033514516","https://openalex.org/W2148935754","https://openalex.org/W7099459805","https://openalex.org/W2055755944","https://openalex.org/W2167926541"],"abstract_inverted_index":{"Context:":[0],"Families":[1,173],"of":[2,6,53,56,84,94,117,121,142,159,174,187,205,220],"experiments":[3,7,85,118,175],"(i.e.,":[4],"groups":[5],"with":[8,125,179],"the":[9,14,40,54,77,90,185,201],"same":[10],"goal)":[11],"are":[12,176],"on":[13],"rise":[15],"in":[16,49,86,119],"Software":[17],"Engineering":[18],"(SE).":[19],"Selecting":[20],"unsuitable":[21,180],"aggregation":[22,59,78,96,133,166,181],"techniques":[23,41,60,79,134,167,182],"to":[24,31,43,61,75,81,99,112,127,138,164,184,199,213,216],"analyze":[25,82,113,139,217],"families":[26,48,83,116,141,219],"may":[27],"undermine":[28],"their":[29],"potential":[30],"provide":[32,109],"in-depth":[33],"insights":[34],"from":[35],"experiments'":[36,45],"results.":[37,207],"Objectives:":[38],"Identifying":[39],"used":[42,80,137,163],"aggregate":[44],"results":[46],"within":[47,65,171],"SE.":[50,87],"Raising":[51],"awareness":[52],"importance":[55],"applying":[57],"suitable":[58,215],"reach":[62],"reliable":[63],"conclusions":[64],"families.":[66,172],"Method:":[67],"We":[68,88,108],"conduct":[69],"a":[70],"systematic":[71],"mapping":[72],"study":[73],"(SMS)":[74],"identify":[76],"outline":[89],"advantages":[91],"and":[92,106,114,157,203,209],"disadvantages":[93],"each":[95],"technique":[97],"according":[98,183],"mature":[100,188],"experimental":[101,189],"disciplines":[102],"such":[103],"as":[104],"medicine":[105],"pharmacology.":[107],"preliminary":[110],"recommendations":[111],"report":[115],"view":[120],"families'":[122],"common":[123],"limitations":[124],"regard":[126],"joint":[128,206],"data":[129],"analysis.":[130],"Results:":[131],"Several":[132],"have":[135],"been":[136],"SE":[140,218],"experiments,":[143],"including":[144],"Narrative":[145],"synthesis,":[146],"Aggregated":[147],"Data":[148,152,192],"(AD),":[149],"Individual":[150],"Participant":[151],"(IPD)":[153],"mega-trial":[154],"or":[155],"stratified,":[156],"Aggregation":[158],"p-values.":[160],"The":[161],"rationale":[162],"select":[165],"is":[168],"rarely":[169],"discussed":[170],"commonly":[177],"analyzed":[178],"literature":[186],"disciplines.":[190],"Conclusion:":[191],"analysis'":[193],"reporting":[194],"practices":[195],"should":[196],"be":[197,214],"improved":[198],"increase":[200],"reliability":[202],"transparency":[204],"AD":[208],"IPD":[210],"stratified":[211],"appear":[212],"experiments.":[221]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2024,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2018-06-01T00:00:00"}