{"id":"https://openalex.org/W2965330732","doi":"https://doi.org/10.1007/s10664-019-09747-0","title":"Meta-analysis for families of experiments in software engineering: a systematic review and reproducibility and validity assessment","display_name":"Meta-analysis for families of experiments in software engineering: a systematic review and reproducibility and validity assessment","publication_year":2019,"publication_date":"2019-07-30","ids":{"openalex":"https://openalex.org/W2965330732","doi":"https://doi.org/10.1007/s10664-019-09747-0","mag":"2965330732"},"language":"en","primary_location":{"id":"doi:10.1007/s10664-019-09747-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-019-09747-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-019-09747-0.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10664-019-09747-0.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012102325","display_name":"Barbara Kitchenham","orcid":"https://orcid.org/0000-0002-6134-8460"},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Barbara Kitchenham","raw_affiliation_strings":["School of Computing and Mathematics, Keele University, Keele, Staffordshire, ST5 5BG, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Mathematics, Keele University, Keele, Staffordshire, ST5 5BG, UK","institution_ids":["https://openalex.org/I56007636"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081451369","display_name":"Lech Madeyski","orcid":"https://orcid.org/0000-0003-3907-3357"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":true,"raw_author_name":"Lech Madeyski","raw_affiliation_strings":["Faculty of Computer Science and Management, Wroclaw University of Science and Technology, Wyb. Wyspianskiego 27, 50-370, Wroclaw, Poland"],"raw_orcid":"https://orcid.org/0000-0003-3907-3357","affiliations":[{"raw_affiliation_string":"Faculty of Computer Science and Management, Wroclaw University of Science and Technology, Wyb. Wyspianskiego 27, 50-370, Wroclaw, Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007624202","display_name":"Pearl Brereton","orcid":null},"institutions":[{"id":"https://openalex.org/I56007636","display_name":"Keele University","ror":"https://ror.org/00340yn33","country_code":"GB","type":"education","lineage":["https://openalex.org/I56007636"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Pearl Brereton","raw_affiliation_strings":["School of Computing and Mathematics, Keele University, Keele, Staffordshire, ST5 5BG, UK"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Mathematics, Keele University, Keele, Staffordshire, ST5 5BG, UK","institution_ids":["https://openalex.org/I56007636"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5081451369"],"corresponding_institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":8.1042,"has_fulltext":true,"cited_by_count":37,"citation_normalized_percentile":{"value":0.97496273,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"25","issue":"1","first_page":"353","last_page":"401"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10430","display_name":"Software Engineering Techniques and Practices","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/meta-analysis","display_name":"Meta-analysis","score":0.7548490166664124},{"id":"https://openalex.org/keywords/crossover","display_name":"Crossover","score":0.5613265633583069},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.5527908205986023},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5161905884742737},{"id":"https://openalex.org/keywords/rounding","display_name":"Rounding","score":0.4904491603374481},{"id":"https://openalex.org/keywords/descriptive-statistics","display_name":"Descriptive statistics","score":0.4886534512042999},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.48535314202308655},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.26321637630462646},{"id":"https://openalex.org/keywords/medline","display_name":"MEDLINE","score":0.2214002013206482},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.18295392394065857},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1611325740814209},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.14720186591148376},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.06514239311218262}],"concepts":[{"id":"https://openalex.org/C95190672","wikidata":"https://www.wikidata.org/wiki/Q815382","display_name":"Meta-analysis","level":2,"score":0.7548490166664124},{"id":"https://openalex.org/C122507166","wikidata":"https://www.wikidata.org/wiki/Q628906","display_name":"Crossover","level":2,"score":0.5613265633583069},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.5527908205986023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5161905884742737},{"id":"https://openalex.org/C136625980","wikidata":"https://www.wikidata.org/wiki/Q663208","display_name":"Rounding","level":2,"score":0.4904491603374481},{"id":"https://openalex.org/C39896193","wikidata":"https://www.wikidata.org/wiki/Q380344","display_name":"Descriptive statistics","level":2,"score":0.4886534512042999},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.48535314202308655},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.26321637630462646},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.2214002013206482},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.18295392394065857},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1611325740814209},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.14720186591148376},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.06514239311218262},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1007/s10664-019-09747-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-019-09747-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-019-09747-0.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:eprints.keele.ac.uk:6635","is_oa":true,"landing_page_url":"http://eprints.keele.ac.uk/6635/1/Kitchenham2019_Article_Meta-analysisForFamiliesOfExpe.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306402149","display_name":"Keele Research Repository (Keele University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I56007636","host_organization_name":"Keele University","host_organization_lineage":["https://openalex.org/I56007636"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1007/s10664-019-09747-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10664-019-09747-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10664-019-09747-0.pdf","source":{"id":"https://openalex.org/S109852484","display_name":"Empirical Software Engineering","issn_l":"1382-3256","issn":["1382-3256","1573-7616"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Empirical Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322733","display_name":"Ministerstwo Edukacji i Nauki","ror":"https://ror.org/05dwvd537"},{"id":"https://openalex.org/F4320323564","display_name":"Politechnika Wroc\u0142awska","ror":"https://ror.org/008fyn775"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2965330732.pdf","grobid_xml":"https://content.openalex.org/works/W2965330732.grobid-xml"},"referenced_works_count":50,"referenced_works":["https://openalex.org/W71228264","https://openalex.org/W404240460","https://openalex.org/W590228970","https://openalex.org/W1511920456","https://openalex.org/W1562193183","https://openalex.org/W1948497384","https://openalex.org/W1964258385","https://openalex.org/W1988962172","https://openalex.org/W1989733111","https://openalex.org/W1998686495","https://openalex.org/W1999649023","https://openalex.org/W2007969951","https://openalex.org/W2010267787","https://openalex.org/W2016999376","https://openalex.org/W2025116371","https://openalex.org/W2026347663","https://openalex.org/W2045116160","https://openalex.org/W2059523379","https://openalex.org/W2070300937","https://openalex.org/W2076029586","https://openalex.org/W2077924088","https://openalex.org/W2087898864","https://openalex.org/W2097672669","https://openalex.org/W2099545896","https://openalex.org/W2126930838","https://openalex.org/W2131154789","https://openalex.org/W2137070227","https://openalex.org/W2139168999","https://openalex.org/W2167926541","https://openalex.org/W2230738057","https://openalex.org/W2253427414","https://openalex.org/W2295373235","https://openalex.org/W2424693453","https://openalex.org/W2562248917","https://openalex.org/W2612764117","https://openalex.org/W2772922594","https://openalex.org/W2780802628","https://openalex.org/W2797120013","https://openalex.org/W2804239311","https://openalex.org/W2887834142","https://openalex.org/W2936534160","https://openalex.org/W2965330732","https://openalex.org/W4229570919","https://openalex.org/W4229753260","https://openalex.org/W4238354999","https://openalex.org/W4241729207","https://openalex.org/W4291732983","https://openalex.org/W4297918764","https://openalex.org/W4298230080","https://openalex.org/W4301029050"],"related_works":["https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W4220780102","https://openalex.org/W2410881844","https://openalex.org/W3196334750","https://openalex.org/W1502401885","https://openalex.org/W2087830269","https://openalex.org/W3106281778"],"abstract_inverted_index":{"Previous":[0],"studies":[1,111],"have":[2],"raised":[3],"concerns":[4],"about":[5],"the":[6,49,83,88,94,97,155],"analysis":[7],"and":[8,13,26,44,46,51,91,177,182],"meta-analysis":[9,85,98,140],"of":[10,17,20,31,53,62,66,96,101,145,180,186,192],"crossover":[11,24,194],"experiments":[12,21,32,67,181],"we":[14,106,131,137,172],"were":[15,132],"aware":[16],"several":[18],"families":[19,30,65,179],"that":[22,33,74],"used":[23,34],"designs":[25],"meta-analysis.":[27,78],"To":[28,142,168],"identify":[29],"meta-analysis,":[35],"to":[36,47,76,81,127,134,188],"investigate":[37],"their":[38,54],"methods":[39],"for":[40,175],"effect":[41],"size":[42],"construction":[43],"aggregation,":[45],"assess":[48],"reproducibility":[50,144],"validity":[52,95],"results.":[55,141],"We":[56,79],"performed":[57],"a":[58,183,190],"systematic":[59],"review":[60],"(SR)":[61],"papers":[63],"reporting":[64,176],"in":[68,148],"high":[69],"quality":[70],"software":[71,165],"engineering":[72,166],"journals,":[73],"attempted":[75,80],"apply":[77],"reproduce":[82,135],"reported":[84],"results":[86],"using":[87],"descriptive":[89],"statistics":[90],"also":[92],"investigated":[93],"process.":[99],"Out":[100],"13":[102],"identified":[103],"primary":[104],"studies,":[105],"reproduced":[107,125],"only":[108],"five.":[109],"Seven":[110],"could":[112,122],"not":[113,123,161],"be":[114,124],"reproduced.":[115],"One":[116],"study":[117],"which":[118],"was":[119],"correctly":[120],"analyzed":[121],"due":[126],"rounding":[128],"errors.":[129],"When":[130],"unable":[133],"results,":[136],"provide":[138],"revised":[139],"support":[143,169],"analyses":[146],"presented":[147],"our":[149],"paper,":[150],"it":[151],"is":[152,160],"complemented":[153],"by":[154,164],"reproducer":[156],"R":[157],"package.":[158],"Meta-analysis":[159],"well":[162],"understood":[163],"researchers.":[167],"novice":[170],"researchers,":[171],"present":[173],"recommendations":[174],"meta-analyzing":[178],"detailed":[184],"example":[185],"how":[187],"analyze":[189],"family":[191],"4-group":[193],"experiments.":[195]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":6},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}