{"id":"https://openalex.org/W2975878921","doi":"https://doi.org/10.1145/3341981.3358959","title":"Statistical Significance Testing in Theory and in Practice","display_name":"Statistical Significance Testing in Theory and in Practice","publication_year":2019,"publication_date":"2019-09-26","ids":{"openalex":"https://openalex.org/W2975878921","doi":"https://doi.org/10.1145/3341981.3358959","mag":"2975878921"},"language":"en","primary_location":{"id":"doi:10.1145/3341981.3358959","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3341981.3358959","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3341981.3358959","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3341981.3358959","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078123101","display_name":"Ben Carterette","orcid":"https://orcid.org/0000-0001-9538-047X"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ben Carterette","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5078123101"],"corresponding_institution_ids":["https://openalex.org/I4210122154"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.08967425,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"257","last_page":"259"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9643999934196472,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9643999934196472,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.957099974155426,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9483000040054321,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6699181795120239},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.6043419241905212},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5948824882507324},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5234641432762146},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.48192858695983887},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.4453064501285553},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.224166601896286},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13033905625343323},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.1283879280090332}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6699181795120239},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.6043419241905212},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5948824882507324},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5234641432762146},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.48192858695983887},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.4453064501285553},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.224166601896286},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13033905625343323},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.1283879280090332},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3341981.3358959","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3341981.3358959","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3341981.3358959","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3341981.3358959","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3341981.3358959","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3341981.3358959","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2019 ACM SIGIR International Conference on Theory of Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5581414644","display_name":"CAREER: Measuring Search Engines' Ability to Help Users Complete Tasks","funder_award_id":"1350799","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7481567382","display_name":null,"funder_award_id":"IIS-1350799","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2975878921.pdf","grobid_xml":"https://content.openalex.org/works/W2975878921.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W643855938","https://openalex.org/W1514893825","https://openalex.org/W1891850176","https://openalex.org/W1967388327","https://openalex.org/W1987690005","https://openalex.org/W1989373893","https://openalex.org/W1990190154","https://openalex.org/W2006996936","https://openalex.org/W2015338694","https://openalex.org/W2017292914","https://openalex.org/W2017826930","https://openalex.org/W2022995284","https://openalex.org/W2023217403","https://openalex.org/W2032577510","https://openalex.org/W2038290376","https://openalex.org/W2064459117","https://openalex.org/W2075893676","https://openalex.org/W2081814119","https://openalex.org/W2110787577","https://openalex.org/W2111310810","https://openalex.org/W2120966432","https://openalex.org/W2133104961","https://openalex.org/W2160892561","https://openalex.org/W2287148974","https://openalex.org/W2331384579","https://openalex.org/W2338477879","https://openalex.org/W2516238915","https://openalex.org/W2560471521","https://openalex.org/W4250181299","https://openalex.org/W4250212051","https://openalex.org/W4253292190","https://openalex.org/W6603117940"],"related_works":["https://openalex.org/W641782856","https://openalex.org/W24443521","https://openalex.org/W2471323292","https://openalex.org/W2296759459","https://openalex.org/W4205202004","https://openalex.org/W3025127592","https://openalex.org/W2188047296","https://openalex.org/W2352527711","https://openalex.org/W9484388","https://openalex.org/W2377100367"],"abstract_inverted_index":{"The":[0],"past":[1],"25":[2],"years":[3],"have":[4,204],"seen":[5],"a":[6,77,110,117,125,130,166,174],"great":[7],"improvement":[8,128],"in":[9,172,186,209,218],"the":[10,41,53,135,215,219,233],"rigor":[11],"of":[12,44,56,154],"experimentation":[13],"on":[14,48,89,116],"information":[15,90],"access":[16,91],"problems.":[17],"This":[18],"is":[19,145,181,194,222],"due":[20],"primarily":[21],"to":[22,60,68,123,142,149,190],"three":[23],"factors:":[24],"high-quality,":[25],"public,":[26],"portable":[27],"test":[28,112],"collections":[29],"such":[30,94,182],"as":[31,95,134,165,237],"those":[32],"produced":[33],"by":[34],"TREC":[35],"(the":[36],"Text":[37],"REtreval":[38],"Conference~\\citetrecbook":[39],"),":[40],"increased":[42,54],"ease":[43],"online":[45,115],"A/B":[46],"testing":[47,59],"large":[49,118],"user":[50,119],"populations,":[51],"and":[52,85,97,121,161,173,192],"practice":[55],"statistical":[57],"hypothesis":[58],"determine":[61],"whether":[62],"observed":[63],"improvements":[64],"can":[65,203],"be":[66,101,143,150,232],"ascribed":[67],"something":[69],"other":[70],"than":[71],"random":[72],"chance.":[73],"Together":[74],"these":[75],"create":[76],"very":[78],"useful":[79,144],"standard":[80],"for":[81,206,235],"reviewers,":[82],"program":[83],"committees,":[84],"journal":[86],"editors;":[87],"work":[88],"(IA)":[92],"problems":[93],"search":[96],"recommendation":[98],"increasingly":[99],"cannot":[100],"published":[102,195,225],"unless":[103],"it":[104],"has":[105,212],"been":[106],"evaluated":[107],"offline":[108],"using":[109,198],"well-constructed":[111],"collection":[113],"or":[114,196],"base":[120],"shown":[122],"produce":[124],"statistically":[126],"significant":[127],"over":[129],"good":[131],"baseline.":[132],"But,":[133],"saying":[136],"goes,":[137],"any":[138],"tool":[139],"sharp":[140,147],"enough":[141,148],"also":[146],"dangerous.":[151],"Statistical":[152],"tests":[153],"significance":[155,180],"are":[156,228],"widely":[157],"misunderstood.":[158],"Most":[159],"researchers":[160],"developers":[162],"treat":[163],"them":[164],"\"black":[167],"box'':":[168],"evaluation":[169],"results":[170],"go":[171],"p-value":[175],"comes":[176],"out.":[177],"But":[178],"because":[179],"an":[183],"important":[184],"factor":[185],"determining":[187],"what":[188,193],"directions":[189],"explore":[191],"deployed,":[197],"p-values":[199],"obtained":[200],"without":[201],"thought":[202],"consequences":[205],"everyone":[207],"working":[208],"IA.":[210],"Ioannidis":[211],"argued":[213],"that":[214,223,231],"main":[216],"consequence":[217],"biomedical":[220],"sciences":[221],"most":[224],"research":[226],"findings":[227],"false;":[229],"could":[230],"case":[234],"IA":[236],"well?":[238]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}