{"id":"https://openalex.org/W3155728719","doi":"https://doi.org/10.1145/3404835.3463236","title":"WWW3E8: 259,000 Relevance Labels for Studying the Effect of Document Presentation Order for Relevance Assessors","display_name":"WWW3E8: 259,000 Relevance Labels for Studying the Effect of Document Presentation Order for Relevance Assessors","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3155728719","doi":"https://doi.org/10.1145/3404835.3463236","mag":"3155728719"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463236","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463236","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091105421","display_name":"Sijie Tao","orcid":"https://orcid.org/0000-0002-6751-5303"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sijie Tao","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074501708","display_name":"Zhaohao Zeng","orcid":"https://orcid.org/0000-0002-9275-8526"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Zhaohao Zeng","raw_affiliation_strings":["Waseda University, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"Waseda University, Tokyo, Japan","institution_ids":["https://openalex.org/I150744194"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":0.2754,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.58138383,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"2376","last_page":"2382"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9873999953269958,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9873999953269958,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9847000241279602,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11550","display_name":"Text and Document Classification Technologies","score":0.9811000227928162,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.8003029823303223},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7974981069564819},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.7406964898109436},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6392275094985962},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.5863343477249146},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.573695182800293},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.4917354881763458},{"id":"https://openalex.org/keywords/popularity","display_name":"Popularity","score":0.46522805094718933},{"id":"https://openalex.org/keywords/order","display_name":"Order (exchange)","score":0.4568290710449219},{"id":"https://openalex.org/keywords/statistic","display_name":"Statistic","score":0.4488796889781952},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.37170684337615967},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3425605893135071},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3222012221813202},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.26412510871887207},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10016006231307983},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08444753289222717}],"concepts":[{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.8003029823303223},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7974981069564819},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.7406964898109436},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6392275094985962},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.5863343477249146},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.573695182800293},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.4917354881763458},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.46522805094718933},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.4568290710449219},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.4488796889781952},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37170684337615967},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3425605893135071},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3222012221813202},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26412510871887207},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10016006231307983},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08444753289222717},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3404835.3463236","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463236","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":22,"referenced_works":["https://openalex.org/W1523019309","https://openalex.org/W1554394657","https://openalex.org/W1824124613","https://openalex.org/W1968927634","https://openalex.org/W1991774506","https://openalex.org/W2008285050","https://openalex.org/W2075893676","https://openalex.org/W2134210222","https://openalex.org/W2137274315","https://openalex.org/W2137843624","https://openalex.org/W2141653139","https://openalex.org/W2154884126","https://openalex.org/W2336806308","https://openalex.org/W2464171116","https://openalex.org/W2609795610","https://openalex.org/W2896022497","https://openalex.org/W2955792106","https://openalex.org/W3035278698","https://openalex.org/W4210764005","https://openalex.org/W4234071252","https://openalex.org/W4300410021","https://openalex.org/W6765590280"],"related_works":["https://openalex.org/W2368605798","https://openalex.org/W2348524959","https://openalex.org/W2518037665","https://openalex.org/W2368049389","https://openalex.org/W2384861574","https://openalex.org/W2952704802","https://openalex.org/W4294565801","https://openalex.org/W2142306706","https://openalex.org/W308652608","https://openalex.org/W2614084750"],"abstract_inverted_index":{"In":[0],"IR":[1,107],"evaluation":[2],"based":[3],"on":[4],"depth-k":[5,94],"pooling,":[6],"there":[7],"are":[8],"several":[9],"strategies":[10,99,124],"to":[11,25,60,112,117,251],"order":[12,29],"the":[13,21,28,51,61,64,78,87,93,106,157,162,171,179,201,205,212,215,226,231,237,254],"pooled":[14,79],"documents":[15,80],"for":[16,102,156],"relevance":[17,198],"assessors.":[18],"Among":[19],"them,":[20],"simplest":[22],"approach":[23,54,66,181],"is":[24,56,63],"completely":[26],"randomise":[27],"\"so":[30],"assessors":[31],"cannot":[32],"tell":[33],"if":[34],"a":[35,70,83,114,143,194,248],"document":[36,91,242],"was":[37],"highly":[38],"ranked":[39],"by":[40,68,81],"some":[41],"system":[42],"or":[43],"how":[44],"many":[45],"systems":[46],"(or":[47],"which":[48],"systems)":[49],"retrieved":[50],"document.\"":[52],"An":[53],"that":[55,85],"in":[57,230,236],"sharp":[58],"contrast":[59],"above":[62,238],"prioritisation":[65,180],"taken":[67],"NTCIRPOOL,":[69],"tool":[71],"widely":[72],"used":[73,229],"at":[74],"NTCIR.":[75],"NTCIRPOOL":[76],"sorts":[77],"\"pseudorelevance,\"":[82],"statistic":[84],"reflects":[86],"popularity":[88],"of":[89,121,138,154,161,182,196,225],"each":[90,120],"within":[92],"pools.":[95],"Although":[96],"these":[97,122],"two":[98,104,123,255],"have":[100,141],"coexisted":[101],"over":[103],"decades,":[105],"research":[108,239],"community":[109],"has":[110],"yet":[111],"reach":[113],"consensus":[115],"as":[116,247],"what":[118],"advantages":[119],"actually":[125],"offer.":[126],"To":[127],"help":[128],"researchers":[129,234],"directly":[130,252],"address":[131],"this":[132],"question":[133,240],"using":[134,170,178],"their":[135],"favourite":[136],"methods":[137],"analysis,":[139],"we":[140],"released":[142],"large-scale":[144],"data":[145,202],"set":[146,203],"called":[147],"WWW3E8.":[148],"It":[149],"comprises":[150],"eight":[151],"independent":[152],"sets":[153],"qrels":[155,167,185],"160":[158],"English":[159,207],"topics":[160],"NTCIR-15":[163],"WWW-3":[164,213],"task:":[165],"four":[166,176],"files":[168,210],"constructed":[169,177],"randomisation":[172],"approach,":[173],"and":[174,217,221],"another":[175],"NTCIRPOOL.":[183],"Each":[184],"file":[186],"covers":[187],"32,375":[188],"topic-document":[189],"pairs;":[190],"hence,":[191],"WWW3E8":[192,246],"contains":[193,204],"total":[195],"259,000":[197],"labels.":[199],"Moreover,":[200],"raw":[206],"subtask":[208],"run":[209],"from":[211],"task,":[214],"randomised":[216],"prioritised":[218],"pool":[219],"files,":[220],"topic-by-run":[222],"score":[223],"matrices":[224],"official":[227],"measures":[228],"task.":[232],"Hence,":[233],"interested":[235],"regarding":[241],"ordering":[243],"can":[244],"utilise":[245],"common":[249],"ground":[250],"compare":[253],"strategies.":[256]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}