{"id":"https://openalex.org/W4205544037","doi":"https://doi.org/10.1145/3494833","title":"Relevance Assessments for Web Search Evaluation: Should We Randomise or Prioritise the Pooled Documents?","display_name":"Relevance Assessments for Web Search Evaluation: Should We Randomise or Prioritise the Pooled Documents?","publication_year":2022,"publication_date":"2022-01-11","ids":{"openalex":"https://openalex.org/W4205544037","doi":"https://doi.org/10.1145/3494833"},"language":"en","primary_location":{"id":"doi:10.1145/3494833","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3494833","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3494833","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3494833","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Okubo, Shinjuku, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University, Okubo, Shinjuku, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091105421","display_name":"Sijie Tao","orcid":"https://orcid.org/0000-0002-6751-5303"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sijie Tao","raw_affiliation_strings":["Waseda University, Okubo, Shinjuku, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6751-5303","affiliations":[{"raw_affiliation_string":"Waseda University, Okubo, Shinjuku, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074501708","display_name":"Zhaohao Zeng","orcid":"https://orcid.org/0000-0002-9275-8526"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Zhaohao Zeng","raw_affiliation_strings":["Waseda University, Okubo, Shinjuku, Tokyo, Japan"],"raw_orcid":"https://orcid.org/0000-0002-9275-8526","affiliations":[{"raw_affiliation_string":"Waseda University, Okubo, Shinjuku, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":1.2743,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.83168419,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":"40","issue":"4","first_page":"1","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9941999912261963,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.7604206800460815},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7313908338546753},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6559221148490906},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5809476375579834},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5689674019813538},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.528874397277832},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.487545907497406},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.4758993089199066},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.34106510877609253},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2336897850036621},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.07574647665023804},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.05580976605415344}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.7604206800460815},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7313908338546753},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6559221148490906},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5809476375579834},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5689674019813538},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.528874397277832},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.487545907497406},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.4758993089199066},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.34106510877609253},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2336897850036621},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.07574647665023804},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.05580976605415344},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3494833","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3494833","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3494833","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3494833","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3494833","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3494833","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4205544037.pdf","grobid_xml":"https://content.openalex.org/works/W4205544037.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1488210821","https://openalex.org/W1507150160","https://openalex.org/W1824124613","https://openalex.org/W1975879668","https://openalex.org/W1991774506","https://openalex.org/W2001142465","https://openalex.org/W2008285050","https://openalex.org/W2057495142","https://openalex.org/W2071446250","https://openalex.org/W2075893676","https://openalex.org/W2088901214","https://openalex.org/W2120308175","https://openalex.org/W2134210222","https://openalex.org/W2137274315","https://openalex.org/W2137843624","https://openalex.org/W2141653139","https://openalex.org/W2154884126","https://openalex.org/W2336806308","https://openalex.org/W2515377091","https://openalex.org/W2609795610","https://openalex.org/W2896022497","https://openalex.org/W2901234380","https://openalex.org/W2955792106","https://openalex.org/W2967489092","https://openalex.org/W2981180456","https://openalex.org/W3117895913","https://openalex.org/W3146722042","https://openalex.org/W3155728719","https://openalex.org/W4234071252","https://openalex.org/W4243617875","https://openalex.org/W6606138943"],"related_works":["https://openalex.org/W2953234277","https://openalex.org/W2626256601","https://openalex.org/W2900413183","https://openalex.org/W147410782","https://openalex.org/W4287804464","https://openalex.org/W3022252430","https://openalex.org/W3103989898","https://openalex.org/W2041353081","https://openalex.org/W1970551129","https://openalex.org/W2581240705"],"abstract_inverted_index":{"In":[0,38],"the":[1,33,53,74,83,116,134,159,170,173,177,183,196,209,238,245,258,277,283,305,317],"context":[2],"of":[3,69,73,100,130,139,187,247,279,311],"depth-":[4],"k":[5],"pooling":[6],"for":[7,21,62],"constructing":[8],"web":[9],"search":[10],"test":[11,273],"collections,":[12,274],"we":[13,48,93,125,275],"compare":[14,94,133],"two":[15,135],"approaches":[16],"to":[17,40,108,115,132,203,226,257,264,292],"ordering":[18],"pooled":[19,306],"documents":[20,165,242,299],"relevance":[22,60,76,215],"assessors:":[23],"The":[24,147],"prioritisation":[25],"strategy":[26,36],"(PRI)":[27],"used":[28],"widely":[29],"at":[30],"NTCIR,":[31],"and":[32,46,51,96,106,180,212,221],"simple":[34],"randomisation":[35],"(RND).":[37],"order":[39,149,161],"address":[41],"research":[42],"questions":[43],"regarding":[44],"PRI":[45,95,178,197,236,284,312],"RND,":[47],"have":[49,288],"constructed":[50,281],"released":[52],"WWW3E8":[54,131],"dataset,":[55],"which":[56,164],"contains":[57],"eight":[58,75],"independent":[59],"labels":[61,77],"32,375":[63],"topic-document":[64],"pairs,":[65],"i.e.,":[66],"a":[67,128,289],"total":[68],"259,000":[70],"labels.":[71],"Four":[72],"were":[78,86],"obtained":[79,87,126],"from":[80,88,304],"PRI-based":[81,214,318],"pools;":[82],"other":[84],"four":[85],"RND-based":[89,230],"pools.":[90,117,259],"Using":[91],"WWW3E8,":[92],"RND":[97,184,210,340],"in":[98,137,344],"terms":[99,138],"inter-assessor":[101,174],"agreement,":[102,105],"system":[103,193],"ranking":[104],"robustness":[107,310],"new":[109,227],"systems":[110,228,269,296,324],"that":[111,181,235,243,253,294,300,316,325,334,339],"did":[112,254],"not":[113,255],"contribute":[114,256],"We":[118],"also":[119,314],"utilise":[120],"an":[121],"assessor":[122],"activity":[123],"log":[124],"as":[127],"byproduct":[129],"strategies":[136],"assessment":[140,155,216],"efficiency.":[141],"Our":[142],"main":[143],"findings":[144],"are:":[145],"(a)":[146],"presentation":[148,160],"has":[150],"no":[151,188,337],"substantial":[152],"impact":[153],"on":[154],"efficiency;":[156],"(b)":[157],"While":[158,308],"substantially":[162,200,220],"affects":[163],"are":[166,199,219,301,320],"judged":[167],"(highly)":[168],"relevant,":[169],"difference":[171],"between":[172],"agreement":[175],"under":[176,182,195,208],"condition":[179,185,198],"is":[186,336,341],"practical":[189],"significance;":[190],"(c)":[191],"Different":[192],"rankings":[194],"more":[201,224],"similar":[202],"one":[204,331],"another":[205],"than":[206,229],"those":[207,252,280],"condition;":[211],"(d)":[213,233],"files":[217],"(qrels)":[218],"statistically":[222],"significantly":[223],"robust":[225],"ones.":[231],"Finding":[232],"suggests":[234],"helps":[237],"assessors":[239],"identify":[240],"relevant":[241,298,329],"affect":[244],"evaluation":[246],"many":[248],"existing":[249],"systems,":[250],"including":[251],"Hence,":[260],"if":[261],"researchers":[262],"need":[263],"evaluate":[265],"their":[266,295],"current":[267],"IR":[268,272],"using":[270,282],"legacy":[271],"recommend":[276],"use":[278],"approach":[285],"unless":[286],"they":[287],"good":[290],"reason":[291],"believe":[293],"retrieve":[297,326],"vastly":[302],"different":[303],"documents.":[307],"this":[309,345],"may":[313],"mean":[315],"pools":[319],"biased":[321],"against":[322],"future":[323],"highly":[327],"novel":[328],"documents,":[330],"should":[332],"note":[333],"there":[335],"evidence":[338],"any":[342],"better":[343],"respect.":[346]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}