{"id":"https://openalex.org/W4372272615","doi":"https://doi.org/10.1145/3539618.3592004","title":"Inference at Scale: Significance Testing for Large Search and Recommendation Experiments","display_name":"Inference at Scale: Significance Testing for Large Search and Recommendation Experiments","publication_year":2023,"publication_date":"2023-07-18","ids":{"openalex":"https://openalex.org/W4372272615","doi":"https://doi.org/10.1145/3539618.3592004"},"language":"en","primary_location":{"id":"doi:10.1145/3539618.3592004","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3539618.3592004","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.02461","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016259502","display_name":"Ngozi Ihemelandu","orcid":"https://orcid.org/0000-0002-8468-1581"},"institutions":[{"id":"https://openalex.org/I120156002","display_name":"Boise State University","ror":"https://ror.org/02e3zdp86","country_code":"US","type":"education","lineage":["https://openalex.org/I120156002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ngozi Ihemelandu","raw_affiliation_strings":["Boise State University, Boise, ID, USA"],"raw_orcid":"https://orcid.org/0000-0002-8468-1581","affiliations":[{"raw_affiliation_string":"Boise State University, Boise, ID, USA","institution_ids":["https://openalex.org/I120156002"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015092805","display_name":"Michael D. Ekstrand","orcid":"https://orcid.org/0000-0003-2467-0108"},"institutions":[{"id":"https://openalex.org/I120156002","display_name":"Boise State University","ror":"https://ror.org/02e3zdp86","country_code":"US","type":"education","lineage":["https://openalex.org/I120156002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michael D. Ekstrand","raw_affiliation_strings":["Boise State University, Boise, ID, USA"],"raw_orcid":"https://orcid.org/0000-0003-2467-0108","affiliations":[{"raw_affiliation_string":"Boise State University, Boise, ID, USA","institution_ids":["https://openalex.org/I120156002"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I120156002"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2087","last_page":"2091"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9983999729156494,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9799000024795532,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.679526686668396},{"id":"https://openalex.org/keywords/wilcoxon-signed-rank-test","display_name":"Wilcoxon signed-rank test","score":0.6486043930053711},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.6392532587051392},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6043726801872253},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.5830533504486084},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.5328664779663086},{"id":"https://openalex.org/keywords/statistical-inference","display_name":"Statistical inference","score":0.5209973454475403},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.49257808923721313},{"id":"https://openalex.org/keywords/type-i-and-type-ii-errors","display_name":"Type I and type II errors","score":0.46181902289390564},{"id":"https://openalex.org/keywords/sign","display_name":"Sign (mathematics)","score":0.43113386631011963},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.417414128780365},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.416317343711853},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3944341838359833},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3887259066104889},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37401872873306274},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.17440176010131836},{"id":"https://openalex.org/keywords/mann\u2013whitney-u-test","display_name":"Mann\u2013Whitney U test","score":0.09889644384384155}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.679526686668396},{"id":"https://openalex.org/C206041023","wikidata":"https://www.wikidata.org/wiki/Q1751970","display_name":"Wilcoxon signed-rank test","level":3,"score":0.6486043930053711},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.6392532587051392},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6043726801872253},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.5830533504486084},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.5328664779663086},{"id":"https://openalex.org/C134261354","wikidata":"https://www.wikidata.org/wiki/Q938438","display_name":"Statistical inference","level":2,"score":0.5209973454475403},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.49257808923721313},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.46181902289390564},{"id":"https://openalex.org/C139676723","wikidata":"https://www.wikidata.org/wiki/Q1193832","display_name":"Sign (mathematics)","level":2,"score":0.43113386631011963},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.417414128780365},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.416317343711853},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3944341838359833},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3887259066104889},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37401872873306274},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.17440176010131836},{"id":"https://openalex.org/C12868164","wikidata":"https://www.wikidata.org/wiki/Q1424533","display_name":"Mann\u2013Whitney U test","level":2,"score":0.09889644384384155},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3539618.3592004","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3539618.3592004","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2305.02461","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.02461","pdf_url":"https://arxiv.org/pdf/2305.02461","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.02461","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.02461","pdf_url":"https://arxiv.org/pdf/2305.02461","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.699999988079071}],"awards":[{"id":"https://openalex.org/G5430501557","display_name":null,"funder_award_id":"17-51278","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8997526758","display_name":null,"funder_award_id":"IIS 17-51278","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4372272615.pdf"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W1990190154","https://openalex.org/W2015338694","https://openalex.org/W2015629112","https://openalex.org/W2075893676","https://openalex.org/W2336806308","https://openalex.org/W2796084947","https://openalex.org/W2798550416","https://openalex.org/W2914164530","https://openalex.org/W3040911419","https://openalex.org/W3101793783","https://openalex.org/W3152624702","https://openalex.org/W3156861296","https://openalex.org/W3158689111","https://openalex.org/W3196879927","https://openalex.org/W4242069725","https://openalex.org/W4243617875","https://openalex.org/W4285546553","https://openalex.org/W4288280764","https://openalex.org/W4297971002"],"related_works":["https://openalex.org/W3003558595","https://openalex.org/W4255580322","https://openalex.org/W154531455","https://openalex.org/W2030905385","https://openalex.org/W4249473446","https://openalex.org/W2259063491","https://openalex.org/W3142061843","https://openalex.org/W4387105795","https://openalex.org/W2380849986","https://openalex.org/W1936869636"],"abstract_inverted_index":{"A":[0],"number":[1],"of":[2,39,52,85,189],"information":[3],"retrieval":[4],"studies":[5,21,48],"have":[6,29,50,185],"been":[7],"done":[8],"to":[9,74,164,201],"assess":[10],"which":[11,27,120],"statistical":[12,131],"techniques":[13],"are":[14,22],"appropriate":[15],"for":[16,41,68,110,138,149],"comparing":[17],"systems.":[18],"However,":[19],"these":[20,75],"focused":[23],"on":[24],"TREC-style":[25],"experiments,":[26],"typically":[28,49],"fewer":[30],"than":[31,114],"100":[32],"topics.":[33],"There":[34],"is":[35,63],"no":[36,144],"similar":[37],"line":[38],"work":[40],"large":[42,89,111,150,166,178],"search":[43,90,179],"and":[44,56,91,101,118,157,177],"recommendation":[45,92,176],"experiments;":[46],"such":[47],"thousands":[51],"topics":[53],"or":[54,204],"users":[55],"much":[57],"sparser":[58],"relevance":[59],"judgements,":[60],"so":[61],"it":[62],"not":[64,161],"clear":[65],"if":[66],"recommendations":[67],"analyzing":[69],"traditional":[70],"TREC":[71],"experiments":[72],"apply":[73],"settings.":[76],"In":[77],"this":[78],"paper,":[79],"we":[80],"empirically":[81],"study":[82],"the":[83,99,115,125,130,155,195],"behavior":[84],"significance":[86],"tests":[87,103,132,159,183],"with":[88,124,174],"evaluation":[93,168,180],"data.":[94],"Our":[95,170],"results":[96],"show":[97,104],"that":[98,173],"Wilcoxon":[100,158],"Sign":[102],"significantly":[105],"higher":[106],"Type-1":[107],"error":[108,127],"rates":[109],"sample":[112,140,151],"sizes":[113],"bootstrap,":[116],"randomization":[117],"t-tests,":[119],"were":[121],"more":[122],"consistent":[123],"expected":[126],"rate.":[128],"While":[129],"displayed":[133],"differences":[134],"in":[135,146],"their":[136,147],"power":[137,148],"smaller":[139],"sizes,":[141],"they":[142],"showed":[143],"difference":[145],"sizes.":[152],"We":[153],"recommend":[154],"sign":[156],"should":[160,198],"be":[162,199],"used":[163,200],"analyze":[165],"scale":[167],"results.":[169,193],"result":[171],"demonstrate":[172],"Top-N":[175],"data,":[181],"most":[182],"would":[184],"a":[186],"100%":[187],"chance":[188],"finding":[190],"statistically":[191],"significant":[192],"Therefore,":[194],"effect":[196],"size":[197],"determine":[202],"practical":[203],"scientific":[205],"significance.":[206]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2023-05-07T00:00:00"}