{"id":"https://openalex.org/W3157456408","doi":"https://doi.org/10.1145/3412841.3441947","title":"The wisdom of the rankers","display_name":"The wisdom of the rankers","publication_year":2021,"publication_date":"2021-03-22","ids":{"openalex":"https://openalex.org/W3157456408","doi":"https://doi.org/10.1145/3412841.3441947","mag":"3157456408"},"language":"en","primary_location":{"id":"doi:10.1145/3412841.3441947","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3412841.3441947","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th Annual ACM Symposium on Applied Computing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070408128","display_name":"David Otero","orcid":"https://orcid.org/0000-0003-1139-0449"},"institutions":[{"id":"https://openalex.org/I11019714","display_name":"Universidade da Coru\u00f1a","ror":"https://ror.org/01qckj285","country_code":"ES","type":"education","lineage":["https://openalex.org/I11019714"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"David Otero","raw_affiliation_strings":["Universidade da Coru\u00f1a, A Coru\u00f1a, Spain"],"affiliations":[{"raw_affiliation_string":"Universidade da Coru\u00f1a, A Coru\u00f1a, Spain","institution_ids":["https://openalex.org/I11019714"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046723532","display_name":"Javier Parapar","orcid":"https://orcid.org/0000-0002-5997-8252"},"institutions":[{"id":"https://openalex.org/I11019714","display_name":"Universidade da Coru\u00f1a","ror":"https://ror.org/01qckj285","country_code":"ES","type":"education","lineage":["https://openalex.org/I11019714"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Javier Parapar","raw_affiliation_strings":["Universidade da Coru\u00f1a, A Coru\u00f1a, Spain"],"affiliations":[{"raw_affiliation_string":"Universidade da Coru\u00f1a, A Coru\u00f1a, Spain","institution_ids":["https://openalex.org/I11019714"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056469959","display_name":"\u00c1lvaro Barreiro","orcid":"https://orcid.org/0000-0002-6698-2946"},"institutions":[{"id":"https://openalex.org/I11019714","display_name":"Universidade da Coru\u00f1a","ror":"https://ror.org/01qckj285","country_code":"ES","type":"education","lineage":["https://openalex.org/I11019714"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"\u00c1lvaro Barreiro","raw_affiliation_strings":["Universidade da Coru\u00f1a, A Coru\u00f1a, Spain"],"affiliations":[{"raw_affiliation_string":"Universidade da Coru\u00f1a, A Coru\u00f1a, Spain","institution_ids":["https://openalex.org/I11019714"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5070408128"],"corresponding_institution_ids":["https://openalex.org/I11019714"],"apc_list":null,"apc_paid":null,"fwci":1.406,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.84123711,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"672","last_page":"680"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.927262544631958},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8095145225524902},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6422619819641113},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5542617440223694},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.5456318855285645},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5098008513450623},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.47573375701904297},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4718584716320038},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4673130214214325},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3207774758338928}],"concepts":[{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.927262544631958},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8095145225524902},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6422619819641113},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5542617440223694},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.5456318855285645},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5098008513450623},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47573375701904297},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4718584716320038},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4673130214214325},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3207774758338928},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3412841.3441947","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3412841.3441947","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 36th Annual ACM Symposium on Applied Computing","raw_type":"proceedings-article"},{"id":"pmh:oai:ruc.udc.es:2183/41377","is_oa":false,"landing_page_url":"http://hdl.handle.net/2183/41377","pdf_url":null,"source":{"id":"https://openalex.org/S4306402204","display_name":"RUC (Universidade Da Coru\u00f1a)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3599354747","display_name":null,"funder_award_id":"GPC ED431B 2019/03, ED431G 2019/01","funder_id":"https://openalex.org/F4320326655","funder_display_name":"Xunta de Galicia"}],"funders":[{"id":"https://openalex.org/F4320326655","display_name":"Xunta de Galicia","ror":"https://ror.org/0181xnw06"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W1966835268","https://openalex.org/W1975362300","https://openalex.org/W1977437814","https://openalex.org/W1983595289","https://openalex.org/W1999822363","https://openalex.org/W2009954908","https://openalex.org/W2010181950","https://openalex.org/W2039017419","https://openalex.org/W2053584122","https://openalex.org/W2057495142","https://openalex.org/W2073913493","https://openalex.org/W2075893676","https://openalex.org/W2102698119","https://openalex.org/W2105157020","https://openalex.org/W2124504084","https://openalex.org/W2132957691","https://openalex.org/W2162697159","https://openalex.org/W2165387257","https://openalex.org/W2388783363","https://openalex.org/W2500119194","https://openalex.org/W2559401578","https://openalex.org/W2609795610","https://openalex.org/W2767754357","https://openalex.org/W2810024740","https://openalex.org/W2885726350","https://openalex.org/W2895814542","https://openalex.org/W2901234380","https://openalex.org/W2945127593","https://openalex.org/W2975063679","https://openalex.org/W2981180456","https://openalex.org/W3000571327","https://openalex.org/W3099446234","https://openalex.org/W4206765718","https://openalex.org/W4234510960","https://openalex.org/W4239181501","https://openalex.org/W4246858749","https://openalex.org/W4252076394","https://openalex.org/W4252222626"],"related_works":["https://openalex.org/W2953234277","https://openalex.org/W2626256601","https://openalex.org/W2900413183","https://openalex.org/W4390975304","https://openalex.org/W147410782","https://openalex.org/W3022252430","https://openalex.org/W4287804464","https://openalex.org/W3103989898","https://openalex.org/W3211292372","https://openalex.org/W803346624"],"abstract_inverted_index":{"Information":[0],"Retrieval":[1],"is":[2,7,43,85,132,160,168],"an":[3,240],"area":[4],"where":[5],"evaluation":[6,20],"crucial":[8],"to":[9,72,142,170,231,243,254],"validate":[10],"newly":[11],"proposed":[12],"models.":[13,145],"As":[14],"the":[15,19,32,38,49,73,79,109,113,117,147,153,175,178,208,211,214,233,259],"first":[16],"step":[17],"in":[18],"of":[21,40,51,62,75,108,149,155,177,210,216,261],"models,":[22],"researchers":[23,136,250],"carry":[24],"out":[25],"offline":[26],"experiments":[27],"on":[28,66,152,205,228,258],"specific":[29],"datasets.":[30],"While":[31],"field":[33],"started":[34],"around":[35],"ad-hoc":[36],"search,":[37],"number":[39],"new":[41,52,144],"tasks":[42,47,164],"continuously":[44],"growing.":[45],"These":[46],"demand":[48],"development":[50],"test":[53,101],"collections":[54,141,197,257],"(documents,":[55],"information":[56],"needs,":[57],"and":[58,128,213],"judgments).":[59],"The":[60],"construction":[61],"those":[63,182],"datasets":[64],"relies":[65,204],"expensive":[67],"campaigns":[68],"like":[69],"TREC.":[70],"Due":[71],"size":[74],"modern":[76],"collections,":[77,102],"obtaining":[78],"relevance":[80],"for":[81,163,165,181,194],"each":[82],"document-topic":[83],"pair":[84],"infeasible.":[86],"To":[87],"reduce":[88],"this":[89,187],"cost,":[90],"organizers":[91],"usually":[92],"apply":[93,226],"a":[94,106,161,191],"technique":[95],"called":[96],"pooling.":[97,217],"When":[98],"building":[99,195],"pooled":[100,140,196],"assessors":[103],"only":[104],"judge":[105],"portion":[107],"documents":[110,234],"selected":[111],"among":[112],"participants'":[114],"results.":[115],"Although":[116],"judgments":[118],"will":[119,124],"not":[120],"be":[121,125],"exhaustive,":[122],"they":[123],"sufficiently":[126],"complete":[127],"unbiased":[129],"if":[130],"pooling":[131,150,227],"done":[133],"correctly.":[134],"Therefore,":[135],"may":[137,251],"safely":[138],"use":[139,252],"evaluate":[143,244],"However,":[146],"application":[148,215],"depends":[151],"existence":[154],"participant":[156,222,262],"systems.":[157,263],"This":[158],"need":[159],"handicap":[162],"which":[166],"it":[167,253],"necessary":[169],"release":[171],"training":[172],"data":[173],"before":[174],"celebration":[176],"competition":[179],"or":[180],"with":[183],"few":[184],"participants.":[185],"In":[186],"paper,":[188],"we":[189,224,247],"present":[190],"simple":[192],"method":[193],"when":[198],"such":[199],"restrictions":[200],"exist.":[201],"Our":[202],"proposal":[203],"two":[206],"principles:":[207],"wisdom":[209],"rankers":[212],"By":[218],"creating":[219],"enough":[220],"artificial":[221],"systems,":[223],"can":[225],"their":[229],"results":[230],"select":[232],"that":[235,249],"merit":[236],"human":[237],"assessment.":[238],"Using":[239],"innovative":[241],"approach":[242],"our":[245],"method,":[246],"show":[248],"produce":[255],"high-quality":[256],"absence":[260]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2021-05-10T00:00:00"}