{"id":"https://openalex.org/W2795252046","doi":"https://doi.org/10.1145/3180155.3180161","title":"Statistical errors in software engineering experiments","display_name":"Statistical errors in software engineering experiments","publication_year":2018,"publication_date":"2018-05-27","ids":{"openalex":"https://openalex.org/W2795252046","doi":"https://doi.org/10.1145/3180155.3180161","mag":"2795252046"},"language":"en","primary_location":{"id":"doi:10.1145/3180155.3180161","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3180155.3180161","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063338445","display_name":"Rolando P. Reyes","orcid":"https://orcid.org/0000-0002-7554-5182"},"institutions":[{"id":"https://openalex.org/I118946981","display_name":"Universidad de las Fuerzas Armadas ESPE","ror":"https://ror.org/05j136930","country_code":"EC","type":"education","lineage":["https://openalex.org/I118946981"]},{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["EC","ES"],"is_corresponding":false,"raw_author_name":"Rolando P. Reyes","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid, Madrid, Spain and Universidad de las Fuerzas Armadas ESPE, Sangolqu\u00ed, Ecuador"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid, Madrid, Spain and Universidad de las Fuerzas Armadas ESPE, Sangolqu\u00ed, Ecuador","institution_ids":["https://openalex.org/I118946981","https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004737603","display_name":"\u00d3scar Dieste","orcid":"https://orcid.org/0000-0002-3060-7853"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Oscar Dieste","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065732953","display_name":"Efra\u00edn R. Fonseca","orcid":null},"institutions":[{"id":"https://openalex.org/I118946981","display_name":"Universidad de las Fuerzas Armadas ESPE","ror":"https://ror.org/05j136930","country_code":"EC","type":"education","lineage":["https://openalex.org/I118946981"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Efra\u00edn R. Fonseca","raw_affiliation_strings":["Universidad de las Fuerzas Armadas ESPE, Sangolqu\u00ed, Ecuador"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad de las Fuerzas Armadas ESPE, Sangolqu\u00ed, Ecuador","institution_ids":["https://openalex.org/I118946981"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056117323","display_name":"Nat\u00e1lia Juristo","orcid":"https://orcid.org/0000-0002-2465-7141"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Natalia Juristo","raw_affiliation_strings":["Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.1131,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":{"value":0.9471313,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1195","last_page":"1206"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9865000247955322,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6693357229232788},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5677580237388611},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5238835215568542},{"id":"https://openalex.org/keywords/a-priori-and-a-posteriori","display_name":"A priori and a posteriori","score":0.5173232555389404},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5059115290641785},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4654897451400757},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4650152325630188},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.4467349052429199},{"id":"https://openalex.org/keywords/type-i-and-type-ii-errors","display_name":"Type I and type II errors","score":0.43323618173599243},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3476428985595703},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.33438289165496826},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3237428665161133},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.27100858092308044},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12141120433807373},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1213327944278717}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6693357229232788},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5677580237388611},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5238835215568542},{"id":"https://openalex.org/C75553542","wikidata":"https://www.wikidata.org/wiki/Q178161","display_name":"A priori and a posteriori","level":2,"score":0.5173232555389404},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5059115290641785},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4654897451400757},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4650152325630188},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.4467349052429199},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.43323618173599243},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3476428985595703},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33438289165496826},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3237428665161133},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.27100858092308044},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12141120433807373},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1213327944278717},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3180155.3180161","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3180155.3180161","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 40th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321837","display_name":"Ministerio de Econom\u00eda y Competitividad","ror":"https://ror.org/034900433"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":98,"referenced_works":["https://openalex.org/W613277099","https://openalex.org/W631794585","https://openalex.org/W1515163059","https://openalex.org/W1518226550","https://openalex.org/W1526710119","https://openalex.org/W1539265701","https://openalex.org/W1565188778","https://openalex.org/W1911264425","https://openalex.org/W1921823546","https://openalex.org/W1966758066","https://openalex.org/W1969939902","https://openalex.org/W1971129844","https://openalex.org/W1972587260","https://openalex.org/W1975976188","https://openalex.org/W1977801131","https://openalex.org/W1979290264","https://openalex.org/W1981663026","https://openalex.org/W1990260092","https://openalex.org/W1994745883","https://openalex.org/W1998208137","https://openalex.org/W2001822577","https://openalex.org/W2003409710","https://openalex.org/W2003549823","https://openalex.org/W2010616334","https://openalex.org/W2011863702","https://openalex.org/W2014232144","https://openalex.org/W2018742491","https://openalex.org/W2019048133","https://openalex.org/W2020602307","https://openalex.org/W2025743471","https://openalex.org/W2035661891","https://openalex.org/W2039854894","https://openalex.org/W2040933668","https://openalex.org/W2042190008","https://openalex.org/W2046434103","https://openalex.org/W2054563706","https://openalex.org/W2058806401","https://openalex.org/W2063387237","https://openalex.org/W2068382122","https://openalex.org/W2069825350","https://openalex.org/W2079155912","https://openalex.org/W2090907135","https://openalex.org/W2093366163","https://openalex.org/W2097001189","https://openalex.org/W2098109710","https://openalex.org/W2104301421","https://openalex.org/W2104831091","https://openalex.org/W2104993088","https://openalex.org/W2108696783","https://openalex.org/W2110503149","https://openalex.org/W2110570722","https://openalex.org/W2112433871","https://openalex.org/W2113610359","https://openalex.org/W2115383527","https://openalex.org/W2119142490","https://openalex.org/W2123468567","https://openalex.org/W2124955986","https://openalex.org/W2126165970","https://openalex.org/W2126941999","https://openalex.org/W2128782367","https://openalex.org/W2129246310","https://openalex.org/W2131195347","https://openalex.org/W2131247853","https://openalex.org/W2134691990","https://openalex.org/W2137270184","https://openalex.org/W2140352503","https://openalex.org/W2144981148","https://openalex.org/W2148439873","https://openalex.org/W2149082609","https://openalex.org/W2149236216","https://openalex.org/W2149923129","https://openalex.org/W2150290224","https://openalex.org/W2153967799","https://openalex.org/W2156067883","https://openalex.org/W2157083801","https://openalex.org/W2158767818","https://openalex.org/W2159022299","https://openalex.org/W2161222712","https://openalex.org/W2166616527","https://openalex.org/W2166697993","https://openalex.org/W2168452606","https://openalex.org/W2295296044","https://openalex.org/W2295373235","https://openalex.org/W2331384579","https://openalex.org/W2335213174","https://openalex.org/W2424693453","https://openalex.org/W2508327075","https://openalex.org/W2561257952","https://openalex.org/W2570760970","https://openalex.org/W2750402394","https://openalex.org/W2884313467","https://openalex.org/W2951307730","https://openalex.org/W3148345308","https://openalex.org/W3193581842","https://openalex.org/W4255135804","https://openalex.org/W4285719527","https://openalex.org/W4301914348","https://openalex.org/W4385962213"],"related_works":["https://openalex.org/W2382078898","https://openalex.org/W2999888572","https://openalex.org/W3124080059","https://openalex.org/W1996408732","https://openalex.org/W2980420425","https://openalex.org/W2278145554","https://openalex.org/W2602060687","https://openalex.org/W2162248999","https://openalex.org/W2562680066","https://openalex.org/W4361292580"],"abstract_inverted_index":{"Background:":[0],"Statistical":[1],"concepts":[2],"and":[3,109,207,219,234,239],"techniques":[4],"are":[5,20,126,214,220],"often":[6],"applied":[7],"incorrectly,":[8],"even":[9],"in":[10,28,39,49,55,62,73,78,186,242],"mature":[11],"disciplines":[12,75,191],"such":[13,91],"as":[14,71,171],"medicine":[15],"or":[16],"psychology.":[17],"Surprisingly,":[18],"there":[19,213],"very":[21],"few":[22],"works":[23],"that":[24,162],"study":[25],"statistical":[26,37,47,95,106,147,169,229],"problems":[27,170],"software":[29],"engineering":[30],"(SE).":[31],"Aim:":[32],"Assess":[33],"the":[34,44,84,129,135,167,176,204,228],"existence":[35],"of":[36,83,134,153,183,231],"errors":[38,48,60,70,136],"SE":[40,65,163,177,232,243],"experiments.":[41],"Method:":[42],"Compile":[43],"most":[45],"common":[46],"experimental":[50,173],"disciplines.":[51,174],"Survey":[52],"experiments":[53,125],"published":[54],"ICSE":[56,79],"to":[57,104,128,180,194,202,224],"assess":[58,105],"whether":[59],"occur":[61],"high":[63],"quality":[64,237],"publications.":[66,244],"Results:":[67],"The":[68,132],"same":[69,168],"identified":[72],"others":[74],"were":[76],"found":[77],"experiments,":[80,188],"where":[81,124],"30%":[82],"reviewed":[85],"papers":[86,123],"included":[87],"several":[88],"error":[89,117],"types":[90],"as:":[92],"a)":[93,142,226],"missing":[94,98],"hypotheses,":[96],"b)":[97,150,235],"sample":[99],"size":[100],"calculation,":[101],"c)":[102],"failure":[103],"test":[107],"assumptions,":[108],"d)":[110],"uncorrected":[111],"multiple":[112],"testing.":[113],"This":[114,157],"rather":[115],"large":[116],"rate":[118],"is":[119,200],"greater":[120],"for":[121],"research":[122,164,199],"confined":[127],"validation":[130],"section.":[131],"origin":[133],"can":[137],"be":[138,181],"traced":[139],"back":[140],"to:":[141],"researchers":[143],"not":[144],"having":[145],"sufficient":[146],"training,":[148],"and,":[149],"a":[151,221],"profusion":[152],"exploratory":[154],"research.":[155],"Conclusions:":[156],"paper":[158],"provides":[159],"preliminary":[160],"evidence":[161],"suffers":[165],"from":[166],"other":[172,190],"However,":[175],"community":[178],"appears":[179],"unaware":[182],"any":[184],"shortcomings":[185],"its":[187],"whereas":[189],"work":[192],"hard":[193],"avoid":[195],"these":[196],"threats.":[197],"Further":[198],"necessary":[201],"find":[203],"underlying":[205],"causes":[206],"set":[208],"up":[209],"corrective":[210],"measures,":[211],"but":[212],"some":[215],"potentially":[216],"effective":[217],"actions":[218],"priori":[222],"easy":[223],"implement:":[225],"improve":[227],"training":[230],"researchers,":[233],"enforce":[236],"assessment":[238],"reporting":[240],"guidelines":[241]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}