{"id":"https://openalex.org/W1980161923","doi":"https://doi.org/10.1145/2661829.2661893","title":"Designing Test Collections for Comparing Many Systems","display_name":"Designing Test Collections for Comparing Many Systems","publication_year":2014,"publication_date":"2014-11-03","ids":{"openalex":"https://openalex.org/W1980161923","doi":"https://doi.org/10.1145/2661829.2661893","mag":"1980161923"},"language":"en","primary_location":{"id":"doi:10.1145/2661829.2661893","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661893","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"61","last_page":"70"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11875","display_name":"Statistics Education and Methodologies","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11875","display_name":"Statistics Education and Methodologies","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7307530641555786},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.649602472782135},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6094859838485718},{"id":"https://openalex.org/keywords/type-i-and-type-ii-errors","display_name":"Type I and type II errors","score":0.572219967842102},{"id":"https://openalex.org/keywords/variance","display_name":"Variance (accounting)","score":0.551264226436615},{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.5274256467819214},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.5185942649841309},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.4972696602344513},{"id":"https://openalex.org/keywords/statistical-hypothesis-testing","display_name":"Statistical hypothesis testing","score":0.4697081446647644},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.4533710479736328},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.433987557888031},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4317959249019623},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.42600885033607483},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.39045995473861694},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14517346024513245}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7307530641555786},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.649602472782135},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6094859838485718},{"id":"https://openalex.org/C40696583","wikidata":"https://www.wikidata.org/wiki/Q989120","display_name":"Type I and type II errors","level":2,"score":0.572219967842102},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.551264226436615},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.5274256467819214},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.5185942649841309},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4972696602344513},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.4697081446647644},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4533710479736328},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.433987557888031},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4317959249019623},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.42600885033607483},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.39045995473861694},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14517346024513245},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2661829.2661893","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2661829.2661893","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320310302","display_name":"National Estuarine Research Reserve System","ror":"https://ror.org/05v14bq57"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W137875067","https://openalex.org/W299870200","https://openalex.org/W578092267","https://openalex.org/W1507711745","https://openalex.org/W1948497384","https://openalex.org/W2021856948","https://openalex.org/W2023217403","https://openalex.org/W2026430213","https://openalex.org/W2027255954","https://openalex.org/W2041098697","https://openalex.org/W2051614536","https://openalex.org/W2052569738","https://openalex.org/W2058624977","https://openalex.org/W2093495945","https://openalex.org/W2096623622","https://openalex.org/W2107031757","https://openalex.org/W2112329501","https://openalex.org/W2133104961","https://openalex.org/W2135985057","https://openalex.org/W2137274315","https://openalex.org/W2144712959","https://openalex.org/W2163004772","https://openalex.org/W2294145134","https://openalex.org/W2339562433","https://openalex.org/W2731504087","https://openalex.org/W4211177544","https://openalex.org/W4300870773","https://openalex.org/W6603899496","https://openalex.org/W6605428294","https://openalex.org/W6680411380","https://openalex.org/W6703967046"],"related_works":["https://openalex.org/W3003558595","https://openalex.org/W84084013","https://openalex.org/W3137029931","https://openalex.org/W2883778230","https://openalex.org/W3088276941","https://openalex.org/W2133619163","https://openalex.org/W4255580322","https://openalex.org/W2083241154","https://openalex.org/W154531455","https://openalex.org/W2030905385"],"abstract_inverted_index":{"A":[0],"researcher":[1],"decides":[2],"to":[3,22,31,63,103,252],"build":[4],"a":[5,43,111,198],"test":[6,160,192,253],"collection":[7,193],"for":[8,41,48,79,100,255],"comparing":[9],"her":[10,66],"new":[11],"information":[12],"retrieval":[13],"(IR)":[14],"systems":[15,109],"with":[16,203],"several":[17,80],"state-of-the-art":[18],"baselines.":[19],"She":[20],"wants":[21],"know":[23],"the":[24,55,90,118,124,131,136,139,184,191,209,219,226,230,236,242],"number":[25],"of":[26,97,106,114,221],"topics":[27],"(n)":[28],"she":[29,37],"needs":[30],"create":[32],"in":[33],"advance,":[34],"so":[35],"that":[36,183],"can":[38,156,213,234],"start":[39],"looking":[40],"(say)":[42],"query":[44],"log":[45],"large":[46],"enough":[47],"sampling":[49],"n":[50,240],"good":[51],"topics,":[52,173],"and":[53,70,75,83,129,138,147,229,241,248],"estimating":[54],"relevance":[56,210],"assessment":[57,211],"cost.":[58],"We":[59,86,162],"provide":[60],"practical":[61],"solutions":[62],"researchers":[64,155,233],"like":[65],"using":[67],"power":[68],"analysis":[69,96,228],"sample":[71],"size":[72],"design":[73,157,194],"techniques,":[74],"demonstrate":[76,163],"its":[77],"usefulness":[78],"IR":[81],"tasks":[82,257],"evaluation":[84,167,185],"measures.":[85],"consider":[87],"not":[88],"only":[89],"paired":[91],"t-test":[92],"but":[93],"also":[94],"one-way":[95],"variance":[98,150],"(ANOVA)":[99],"significance":[101],"testing":[102],"accommodate":[104],"comparison":[105],"m(\u2265":[107],"2)":[108],"under":[110],"given":[112],"set":[113,179,220],"statistical":[115,222],"requirements":[116],"(\u03b1:":[117],"Type":[119,125],"I":[120],"error":[121,127],"rate,":[122,128],"\u00df:":[123],"II":[126],"minD:":[130],"minimum":[132],"detectable":[133],"difference":[134],"between":[135,239],"best":[137],"worst":[140],"systems).":[141],"Using":[142],"our":[143],"simple":[144],"Excel":[145],"tools":[146,249],"some":[148],"pooled":[149],"estimates":[151],"from":[152],"past":[153,204],"data,":[154,205],"statistically":[158],"well-designed":[159],"collections.":[161],"that,":[164],"as":[165,258],"different":[166,170,177],"measures":[168,186],"have":[169],"variances":[171],"across":[172],"they":[174],"inevitably":[175],"require":[176],"topic":[178],"sizes.":[180],"This":[181],"suggests":[182],"should":[187],"be":[188,214],"chosen":[189],"at":[190],"phase.":[195],"Moreover,":[196],"through":[197],"pool":[199,243],"depth":[200,244],"reduction":[201],"experiment":[202],"we":[206],"show":[207],"how":[208],"cost":[212,227],"reduced":[215],"dramatically":[216],"while":[217],"freezing":[218],"requirements.":[223],"Based":[224],"on":[225],"available":[231],"budget,":[232],"determine":[235],"right":[237],"balance":[238],"pd.":[245],"Our":[246],"techniques":[247],"are":[250],"applicable":[251],"collections":[254],"non-IR":[256],"well.":[259]},"counts_by_year":[{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":4},{"year":2014,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}