{"id":"https://openalex.org/W4378549003","doi":"https://doi.org/10.1145/3600227","title":"On the Ordering of Pooled Web Pages, Gold Assessments, and Bronze Assessments","display_name":"On the Ordering of Pooled Web Pages, Gold Assessments, and Bronze Assessments","publication_year":2023,"publication_date":"2023-05-27","ids":{"openalex":"https://openalex.org/W4378549003","doi":"https://doi.org/10.1145/3600227"},"language":"en","primary_location":{"id":"doi:10.1145/3600227","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600227","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600227","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3600227","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023595778","display_name":"Tetsuya Sakai","orcid":"https://orcid.org/0000-0002-6720-963X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Tetsuya Sakai","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6720-963X","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091105421","display_name":"Sijie Tao","orcid":"https://orcid.org/0000-0002-6751-5303"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sijie Tao","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0002-6751-5303","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007524571","display_name":"Nuo Chen","orcid":"https://orcid.org/0000-0001-8600-8203"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Nuo Chen","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0000-0001-8600-8203","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100425021","display_name":"Yujing Li","orcid":"https://orcid.org/0009-0002-7204-526X"},"institutions":[{"id":"https://openalex.org/I150744194","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83","country_code":"JP","type":"education","lineage":["https://openalex.org/I150744194"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yujing Li","raw_affiliation_strings":["Waseda University, Japan"],"raw_orcid":"https://orcid.org/0009-0002-7204-526X","affiliations":[{"raw_affiliation_string":"Waseda University, Japan","institution_ids":["https://openalex.org/I150744194"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078639165","display_name":"Maria Maistro","orcid":"https://orcid.org/0000-0002-7001-4817"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Maria Maistro","raw_affiliation_strings":["University of Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-7001-4817","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030377390","display_name":"Zhumin Chu","orcid":"https://orcid.org/0000-0002-1233-0184"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhumin Chu","raw_affiliation_strings":["Tsinghua University, P. R. C"],"raw_orcid":"https://orcid.org/0000-0002-1233-0184","affiliations":[{"raw_affiliation_string":"Tsinghua University, P. R. C","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069843101","display_name":"Nicola Ferro","orcid":"https://orcid.org/0000-0001-9219-6239"},"institutions":[{"id":"https://openalex.org/I138689650","display_name":"University of Padua","ror":"https://ror.org/00240q980","country_code":"IT","type":"education","lineage":["https://openalex.org/I138689650"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Nicola Ferro","raw_affiliation_strings":["University of Padua, Italy"],"raw_orcid":"https://orcid.org/0000-0001-9219-6239","affiliations":[{"raw_affiliation_string":"University of Padua, Italy","institution_ids":["https://openalex.org/I138689650"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5023595778"],"corresponding_institution_ids":["https://openalex.org/I150744194"],"apc_list":null,"apc_paid":null,"fwci":0.3313,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.63445547,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"42","issue":"1","first_page":"1","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9958000183105469,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6974883675575256},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5751842260360718},{"id":"https://openalex.org/keywords/bronze","display_name":"Bronze","score":0.5573282241821289},{"id":"https://openalex.org/keywords/gold-standard","display_name":"Gold standard (test)","score":0.5292108654975891},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5208725929260254},{"id":"https://openalex.org/keywords/gold-alloys","display_name":"Gold alloys","score":0.4575542211532593},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.44695529341697693},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3720548152923584},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.17391321063041687},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16384273767471313},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.1393164098262787}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6974883675575256},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5751842260360718},{"id":"https://openalex.org/C2778478046","wikidata":"https://www.wikidata.org/wiki/Q34095","display_name":"Bronze","level":2,"score":0.5573282241821289},{"id":"https://openalex.org/C40993552","wikidata":"https://www.wikidata.org/wiki/Q514654","display_name":"Gold standard (test)","level":2,"score":0.5292108654975891},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5208725929260254},{"id":"https://openalex.org/C2983538551","wikidata":"https://www.wikidata.org/wiki/Q309191","display_name":"Gold alloys","level":3,"score":0.4575542211532593},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.44695529341697693},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3720548152923584},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.17391321063041687},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16384273767471313},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.1393164098262787},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C2780026712","wikidata":"https://www.wikidata.org/wiki/Q37756","display_name":"Alloy","level":2,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C195244886","wikidata":"https://www.wikidata.org/wiki/Q41493","display_name":"Ancient history","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3600227","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600227","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600227","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:publications/0d1d183d-1473-490d-9b19-2dffa6b494e4","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/0d1d183d-1473-490d-9b19-2dffa6b494e4","pdf_url":"https://curis.ku.dk/ws/files/390398343/3600227.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sakai , T , Tao , S , Chen , N , Li , Y , Maistro , M , Chu , Z & Ferro , N 2023 , ' On the Ordering of Pooled Web Pages, Gold Assessments, and Bronze Assessments ' , ACM Transactions on Information Systems , vol. 42 , no. 1 , 23 . https://doi.org/10.1145/3600227","raw_type":"article"},{"id":"pmh:oai:www.research.unipd.it:11577/3506563","is_oa":false,"landing_page_url":"https://hdl.handle.net/11577/3506563","pdf_url":null,"source":{"id":"https://openalex.org/S4377196283","display_name":"Research Padua  Archive (University of Padua)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I138689650","host_organization_name":"University of Padua","host_organization_lineage":["https://openalex.org/I138689650"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1145/3600227","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3600227","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3600227","source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4894323551","display_name":"Multi-aspect and diffErenTiable Evaluation of Rankings","funder_award_id":"893667","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320322392","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549"},{"id":"https://openalex.org/F4320322638","display_name":"Waseda University","ror":"https://ror.org/00ntfnx83"},{"id":"https://openalex.org/F4320332178","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4378549003.pdf","grobid_xml":"https://content.openalex.org/works/W4378549003.grobid-xml"},"referenced_works_count":37,"referenced_works":["https://openalex.org/W1507150160","https://openalex.org/W1546098095","https://openalex.org/W1967879792","https://openalex.org/W1987690005","https://openalex.org/W1987831316","https://openalex.org/W1991774506","https://openalex.org/W2000289953","https://openalex.org/W2008285050","https://openalex.org/W2013575152","https://openalex.org/W2037140704","https://openalex.org/W2037789405","https://openalex.org/W2039017419","https://openalex.org/W2050467853","https://openalex.org/W2053528567","https://openalex.org/W2057495142","https://openalex.org/W2068098598","https://openalex.org/W2069870183","https://openalex.org/W2071446250","https://openalex.org/W2075893676","https://openalex.org/W2120308175","https://openalex.org/W2122053905","https://openalex.org/W2137843624","https://openalex.org/W2141653139","https://openalex.org/W2150234222","https://openalex.org/W2609795610","https://openalex.org/W2802553787","https://openalex.org/W2895814542","https://openalex.org/W2896022497","https://openalex.org/W2901234380","https://openalex.org/W2912485946","https://openalex.org/W2967489092","https://openalex.org/W2981180456","https://openalex.org/W4205544037","https://openalex.org/W4243617875","https://openalex.org/W4254125302","https://openalex.org/W4255492838","https://openalex.org/W4284704763"],"related_works":["https://openalex.org/W2067067895","https://openalex.org/W2554340036","https://openalex.org/W2388200690","https://openalex.org/W4221009686","https://openalex.org/W2391706077","https://openalex.org/W2381915023","https://openalex.org/W2943563632","https://openalex.org/W2898334687","https://openalex.org/W2058701483","https://openalex.org/W2163707700"],"abstract_inverted_index":{"The":[0,65,110,246],"present":[1],"study":[2,158],"leverages":[3],"a":[4,11,84,95,103],"recent":[5],"opportunity":[6],"we":[7,39],"had":[8],"to":[9,182,220,236,311,325,348,356],"create":[10],"new":[12],"English":[13],"web":[14],"search":[15],"test":[16,35,177,224],"collection":[17,36],"for":[18,73,93,107,259],"the":[19,34,46,58,69,74,99,108,123,143,152,163,170,175,190,221,230,257,273,281,292,301,307,312,326],"NTCIR-16":[20],"We":[21,154],"Want":[22],"Web":[23],"(WWW-4)":[24],"task,":[25],"which":[26,53,130,136,179],"concluded":[27],"in":[28,54,90,161,256,315,354],"June":[29],"2022.":[30],"More":[31],"specifically,":[32],"through":[33,261],"construction":[37],"effort,":[38],"examined":[40],"two":[41],"factors":[42],"that":[43,86,156,162,267,334],"may":[44,56,147,269,345],"affect":[45,57],"relevance":[47],"assessments":[48,187],"of":[49,61,105,165,223,249,300],"depth-":[50],"k":[51],"pools,":[52],"turn":[55],"relative":[59],"evaluation":[60],"different":[62,211],"IR":[63],"systems.":[64],"first":[66],"factor":[67,112],"is":[68,83,113,159,287],"document":[70,274],"ordering":[71,275],"strategy":[72,276],"assessors,":[75],"namely,":[76],"prioritisation":[77],"(PRI)":[78],"and":[79,126,135,146,229,280],"randomisation":[80],"(RND).":[81],"PRI":[82],"method":[85],"has":[87],"been":[88],"used":[89],"NTCIR":[91],"tasks":[92],"over":[94],"decade;":[96],"it":[97,264,344],"ranks":[98],"pooled":[100],"documents":[101,131,302],"by":[102,306],"kind":[104],"pseudorelevance":[106],"assessors.":[109],"second":[111],"assessor":[114,197,282],"type,":[115],"i.e.,":[116],"Gold":[117,120,171,191,241,308,327,336,357],"or":[118],"Bronze.":[119],"assessors":[121,140,172,234,242,294,309,314,337,353,358],"are":[122,132,137,141,252,338],"topic":[124,144],"creators":[125,145],"therefore":[127],"they":[128,360],"\u201cknow\u201d":[129],"(highly)":[133],"relevant":[134,323],"not;":[138],"Bronze":[139,186,233,293,313,352],"not":[142],"lack":[148],"sufficient":[149],"knowledge":[150],"about":[151],"topics.":[153],"believe":[155],"our":[157,250,316],"unique":[160],"authors":[164],"this":[166],"article":[167],"served":[168],"as":[169,253],"when":[173],"creating":[174],"WWW-4":[176],"collection,":[178],"enabled":[180],"us":[181],"closely":[183],"examine":[184,196],"why":[185,232],"differ":[188],"from":[189,340],"ones.":[192],"Our":[193],"research":[194],"questions":[195],"efficiency":[198],"(":[199,204,214,226,243],"RQ1":[200,260],"),":[201,206,216,228],"inter-assessor":[202],"agreement":[203],"RQ2":[205],"system":[207,217],"ranking":[208,218],"similarity":[209],"with":[210],"qrels":[212],"files":[213],"RQ3":[215],"robustness":[219],"choice":[222],"topics":[225],"RQ4":[227,262],"reasons":[231],"tend":[235],"be":[237,346],"more":[238,271],"liberal":[239],"than":[240,272],"RQ5":[244,298],").":[245],"most":[247],"remarkable":[248],"results":[251],"follows:":[254],"First,":[255],"comparisons":[258],",":[263,299],"turned":[265],"out":[266],"what":[268],"matter":[270],"(PRI":[277],"vs.":[278,285],"RND)":[279],"type":[283],"(Gold":[284],"Bronze)":[286],"how":[288],"well-motivated":[289],"and/or":[290],"well-trained":[291],"are.":[295],"Second,":[296],"regarding":[297],"originally":[303],"judged":[304],"nonrelevant":[305],"contrary":[310],"experiments,":[317],"almost":[318],"one":[319],"half":[320],"were":[321],"truly":[322],"according":[324],"assessors\u2019":[328],"own":[329],"reconsiderations.":[330],"This":[331],"result":[332],"suggests":[333],"even":[335],"far":[339],"perfect;":[341],"budget":[342],"permitting,":[343],"beneficial":[347],"hire":[349],"highly":[350],"motivated":[351],"addition":[355],"so":[359],"can":[361],"complement":[362],"each":[363],"other.":[364]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-23T08:51:43.019350","created_date":"2025-10-10T00:00:00"}