{"id":"https://openalex.org/W2017292914","doi":"https://doi.org/10.1145/2094072.2094076","title":"Multiple testing in statistical analysis of systems-based information retrieval experiments","display_name":"Multiple testing in statistical analysis of systems-based information retrieval experiments","publication_year":2012,"publication_date":"2012-02-01","ids":{"openalex":"https://openalex.org/W2017292914","doi":"https://doi.org/10.1145/2094072.2094076","mag":"2017292914"},"language":"en","primary_location":{"id":"doi:10.1145/2094072.2094076","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2094072.2094076","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109487140","display_name":"Benjamin Carterette","orcid":null},"institutions":[{"id":"https://openalex.org/I86501945","display_name":"University of Delaware","ror":"https://ror.org/01sbq1a82","country_code":"US","type":"education","lineage":["https://openalex.org/I86501945"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Benjamin A. Carterette","raw_affiliation_strings":["University of Delaware, Newark, DE"],"affiliations":[{"raw_affiliation_string":"University of Delaware, Newark, DE","institution_ids":["https://openalex.org/I86501945"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5109487140"],"corresponding_institution_ids":["https://openalex.org/I86501945"],"apc_list":null,"apc_paid":null,"fwci":12.8216,"has_fulltext":false,"cited_by_count":143,"citation_normalized_percentile":{"value":0.98707629,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"30","issue":"1","first_page":"1","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9708999991416931,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9708999991416931,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10243","display_name":"Statistical Methods and Bayesian Inference","score":0.9523000121116638,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9488000273704529,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8266544342041016},{"id":"https://openalex.org/keywords/randomness","display_name":"Randomness","score":0.5965240001678467},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.594207763671875},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5392106771469116},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5367048978805542},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.49841785430908203},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.48075056076049805},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.41047748923301697},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.35606086254119873},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.17659541964530945},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.0924500823020935},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.07599595189094543}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8266544342041016},{"id":"https://openalex.org/C125112378","wikidata":"https://www.wikidata.org/wiki/Q176640","display_name":"Randomness","level":2,"score":0.5965240001678467},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.594207763671875},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5392106771469116},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5367048978805542},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.49841785430908203},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.48075056076049805},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.41047748923301697},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35606086254119873},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.17659541964530945},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0924500823020935},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.07599595189094543},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2094072.2094076","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2094072.2094076","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.357.3647","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.357.3647","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ir.cis.udel.edu/~carteret/papers/TOIS-2011-0053.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.648.3056","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.648.3056","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ir.cis.udel.edu/~carteret/papers/mcp.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.4399999976158142,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":52,"referenced_works":["https://openalex.org/W75771211","https://openalex.org/W135190683","https://openalex.org/W137875067","https://openalex.org/W396053900","https://openalex.org/W603193392","https://openalex.org/W1513618424","https://openalex.org/W1546195864","https://openalex.org/W1571975558","https://openalex.org/W1607872540","https://openalex.org/W1990190154","https://openalex.org/W1996908140","https://openalex.org/W2006996936","https://openalex.org/W2010181950","https://openalex.org/W2015338694","https://openalex.org/W2015629112","https://openalex.org/W2017826930","https://openalex.org/W2023217403","https://openalex.org/W2034562813","https://openalex.org/W2035983506","https://openalex.org/W2037140704","https://openalex.org/W2038290376","https://openalex.org/W2045656233","https://openalex.org/W2049083308","https://openalex.org/W2052201263","https://openalex.org/W2073722401","https://openalex.org/W2075893676","https://openalex.org/W2077046902","https://openalex.org/W2081814119","https://openalex.org/W2103151450","https://openalex.org/W2105090664","https://openalex.org/W2111310810","https://openalex.org/W2126843316","https://openalex.org/W2133104961","https://openalex.org/W2136105491","https://openalex.org/W2136887149","https://openalex.org/W2140787629","https://openalex.org/W2144981148","https://openalex.org/W2159267296","https://openalex.org/W2292609460","https://openalex.org/W2324392187","https://openalex.org/W2331384579","https://openalex.org/W2496945004","https://openalex.org/W2502759836","https://openalex.org/W2796677525","https://openalex.org/W3142746689","https://openalex.org/W4232383088","https://openalex.org/W4233760227","https://openalex.org/W4289255550","https://openalex.org/W4301861531","https://openalex.org/W6634144336","https://openalex.org/W6723301277","https://openalex.org/W7043203782"],"related_works":["https://openalex.org/W3034924094","https://openalex.org/W1488708774","https://openalex.org/W3094954546","https://openalex.org/W2981906196","https://openalex.org/W4391100477","https://openalex.org/W4327779705","https://openalex.org/W1513698804","https://openalex.org/W4310560702","https://openalex.org/W4377083194","https://openalex.org/W3039360660"],"abstract_inverted_index":{"High-quality":[0],"reusable":[1,158],"test":[2,159],"collections":[3],"and":[4,126,128,184],"formal":[5],"statistical":[6,138],"hypothesis":[7,139],"testing":[8,56,75],"together":[9],"support":[10],"a":[11,61,77,89,117],"rigorous":[12],"experimental":[13,155,182],"environment":[14],"for":[15,122],"information":[16,81,95],"retrieval":[17,43],"research.":[18],"But":[19],"as":[20],"Armstrong":[21],"et":[22],"al.":[23],"[2009b]":[24],"recently":[25],"argued,":[26],"global":[27],"analysis":[28,123],"of":[29,54,57,64,80,93,119,145,154,176,179],"experiments":[30,121],"suggests":[31],"that":[32,68,106,130,167],"there":[33],"has":[34,148],"actually":[35],"been":[36],"little":[37],"real":[38],"improvement":[39],"in":[40,51,109,181],"ad":[41],"hoc":[42],"effectiveness":[44],"over":[45],"time.":[46],"We":[47,66,112],"investigate":[48],"this":[49,94],"phenomenon":[50],"the":[52,69,83,152,177],"context":[53],"simultaneous":[55],"many":[58,175],"hypotheses":[59],"using":[60,157],"fixed":[62],"set":[63,118],"data.":[65],"argue":[67],"most":[70],"common":[71],"approaches":[72],"to":[73,98,115,141,165],"significance":[74],"ignore":[76],"great":[78],"deal":[79],"about":[82,102],"world.":[84],"Taking":[85],"into":[86],"account":[87],"even":[88],"fairly":[90],"small":[91],"amount":[92],"can":[96,133],"lead":[97],"very":[99,163],"different":[100],"conclusions":[101],"systems":[103],"than":[104],"those":[105],"have":[107,173],"appeared":[108],"published":[110],"literature.":[111],"demonstrate":[113],"how":[114],"model":[116],"IR":[120],"both":[124],"mathematically":[125],"practically,":[127],"show":[129],"doing":[131],"so":[132],"cause":[134],"p":[135],"-values":[136],"from":[137],"tests":[140],"increase":[142],"by":[143],"orders":[144],"magnitude.":[146],"This":[147],"major":[149],"consequences":[150],"on":[151],"interpretation":[153],"results":[156],"collections:":[160],"it":[161],"is":[162,169],"difficult":[164],"conclude":[166],"anything":[168],"significant":[170],"once":[171],"we":[172],"modeled":[174],"sources":[178],"randomness":[180],"design":[183],"analysis.":[185]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":25},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":16},{"year":2020,"cited_by_count":10},{"year":2019,"cited_by_count":14},{"year":2018,"cited_by_count":10},{"year":2017,"cited_by_count":9},{"year":2016,"cited_by_count":7},{"year":2015,"cited_by_count":9},{"year":2014,"cited_by_count":6},{"year":2013,"cited_by_count":9},{"year":2012,"cited_by_count":5}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}