{"id":"https://openalex.org/W2499321295","doi":"https://doi.org/10.1145/2910586","title":"Listwise Learning to Rank from Crowds","display_name":"Listwise Learning to Rank from Crowds","publication_year":2016,"publication_date":"2016-07-20","ids":{"openalex":"https://openalex.org/W2499321295","doi":"https://doi.org/10.1145/2910586","mag":"2499321295"},"language":"en","primary_location":{"id":"doi:10.1145/2910586","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2910586","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000753987","display_name":"Ou Wu","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ou Wu","raw_affiliation_strings":["Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045858387","display_name":"You Qiang","orcid":"https://orcid.org/0000-0002-9225-5689"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiang You","raw_affiliation_strings":["Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100532486","display_name":"Fen Xia","orcid":null},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fen Xia","raw_affiliation_strings":["Baidu, Inc., Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baidu, Inc., Beijing, China","institution_ids":["https://openalex.org/I98301712"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100732507","display_name":"Lei Ma","orcid":"https://orcid.org/0000-0002-5099-5312"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Ma","raw_affiliation_strings":["Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210094879"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114549594","display_name":"Weiming Hu","orcid":"https://orcid.org/0000-0001-9237-8825"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210097554","display_name":"Center for Excellence in Brain Science and Intelligence Technology","ror":"https://ror.org/00vpwhm04","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210097554"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weiming Hu","raw_affiliation_strings":["CAS Center for Excellence in Brain Science and Intelligence Technology and Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CAS Center for Excellence in Brain Science and Intelligence Technology and Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210097554"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.0742,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.91658715,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"11","issue":"1","first_page":"1","last_page":"39"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9706000089645386,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9674000144004822,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9041826128959656},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.8265911340713501},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7917051315307617},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7667556405067444},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.7231903672218323},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.7094136476516724},{"id":"https://openalex.org/keywords/crowds","display_name":"Crowds","score":0.7061150074005127},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6250354051589966},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.48559874296188354},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.47444045543670654},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10120046138763428},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.09737825393676758}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9041826128959656},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.8265911340713501},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7917051315307617},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7667556405067444},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7231903672218323},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.7094136476516724},{"id":"https://openalex.org/C2777852691","wikidata":"https://www.wikidata.org/wiki/Q13430821","display_name":"Crowds","level":2,"score":0.7061150074005127},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6250354051589966},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.48559874296188354},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.47444045543670654},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10120046138763428},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.09737825393676758},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2910586","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2910586","pdf_url":null,"source":{"id":"https://openalex.org/S41523882","display_name":"ACM Transactions on Knowledge Discovery from Data","issn_l":"1556-4681","issn":["1556-4681","1556-472X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Knowledge Discovery from Data","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7812194512","display_name":null,"funder_award_id":"61379098","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W43928053","https://openalex.org/W82775311","https://openalex.org/W96765277","https://openalex.org/W166707811","https://openalex.org/W265767496","https://openalex.org/W1526445072","https://openalex.org/W1541505110","https://openalex.org/W1550469973","https://openalex.org/W1769426539","https://openalex.org/W1934555896","https://openalex.org/W1972675781","https://openalex.org/W1973435495","https://openalex.org/W1974138240","https://openalex.org/W1982173547","https://openalex.org/W1986533012","https://openalex.org/W2006928886","https://openalex.org/W2008244516","https://openalex.org/W2038950446","https://openalex.org/W2049633694","https://openalex.org/W2051275384","https://openalex.org/W2063445365","https://openalex.org/W2066949013","https://openalex.org/W2078264666","https://openalex.org/W2091158010","https://openalex.org/W2103850933","https://openalex.org/W2114523181","https://openalex.org/W2115822807","https://openalex.org/W2117154949","https://openalex.org/W2117853077","https://openalex.org/W2120387782","https://openalex.org/W2122913573","https://openalex.org/W2124192166","https://openalex.org/W2127228711","https://openalex.org/W2129245267","https://openalex.org/W2131427446","https://openalex.org/W2134305421","https://openalex.org/W2144660879","https://openalex.org/W2145680575","https://openalex.org/W2149427297","https://openalex.org/W2154096410","https://openalex.org/W2159852473","https://openalex.org/W2159973364","https://openalex.org/W2160457243","https://openalex.org/W2186184713","https://openalex.org/W2400059445","https://openalex.org/W2405188152","https://openalex.org/W2467426667","https://openalex.org/W3006452435","https://openalex.org/W3015812362","https://openalex.org/W3029645440","https://openalex.org/W4212898980","https://openalex.org/W4226054531","https://openalex.org/W6646904008"],"related_works":["https://openalex.org/W3127142483","https://openalex.org/W3160516639","https://openalex.org/W4385565564","https://openalex.org/W2898073868","https://openalex.org/W2138488530","https://openalex.org/W2798835721","https://openalex.org/W2971071571","https://openalex.org/W2387658907","https://openalex.org/W2922169395","https://openalex.org/W2385796165"],"abstract_inverted_index":{"Learning":[0],"to":[1,31,62,69,77,97,172,209,216,260],"rank":[2,32,70,98,173,210],"has":[3],"received":[4],"great":[5],"attention":[6],"in":[7,16,45,266,282],"recent":[8],"years":[9],"as":[10,20,54],"it":[11,55],"plays":[12],"a":[13,41,73,85,124,131],"crucial":[14],"role":[15],"many":[17,67],"applications":[18],"such":[19],"information":[21,156,179,195,279],"retrieval":[22],"and":[23,82,140,143,198,231,247,287],"data":[24],"mining.":[25],"The":[26,223,252,270],"existing":[27,121,248],"concept":[28],"of":[29,183,193,254,262,272,290],"learning":[30,68,96,134,171,200,208,264],"assumes":[33],"that":[34,236,277],"each":[35],"training":[36,220],"instance":[37],"is":[38,76,115,127,163,186,280],"associated":[39],"with":[40,99,218],"reliable":[42,64,268],"label.":[43],"However,":[44],"practice,":[46],"this":[47,167],"assumption":[48],"does":[49],"not":[50],"necessarily":[51],"hold":[52],"true":[53],"may":[56,107],"be":[57,108],"infeasible":[58],"or":[59],"remarkably":[60],"expensive":[61],"obtain":[63],"labels":[65,79,101,139,176,213],"for":[66],"applications.":[71],"Therefore,":[72,166],"feasible":[74],"approach":[75,241,246],"collect":[78],"from":[80,88,103,174,211],"crowds":[81],"then":[83],"learn":[84],"ranking":[86,113,125,146,221,285],"function":[87,126,147],"crowdsourcing":[89,100,151,212,249],"labels.":[90,269],"This":[91],"study":[92,168],"explores":[93],"the":[94,145,181,206,237,244,255,263,273],"listwise":[95],"obtained":[102],"multiple":[104],"annotators,":[105],"who":[106],"unreliable.":[109],"A":[110],"new":[111],"probabilistic":[112],"model":[114,265],"first":[116],"proposed":[117,224,256],"by":[118,129],"combining":[119],"two":[120],"models.":[122],"Subsequently,":[123],"trained":[128],"proposing":[130],"maximum":[132,238],"likelihood":[133,239],"approach,":[135],"which":[136],"estimates":[137],"ground-truth":[138],"annotator":[141],"expertise,":[142],"trains":[144],"iteratively.":[148],"In":[149,188],"practical":[150],"machine":[152],"learning,":[153],"valuable":[154],"side":[155,178,194,278],"(e.g.,":[157],"professional":[158],"grades)":[159],"about":[160],"involved":[161,184],"annotators":[162,185],"normally":[164],"attainable.":[165],"also":[169],"investigates":[170],"crowd":[175],"when":[177],"on":[180,228],"expertise":[182,288],"available.":[187],"particular,":[189],"three":[190],"basic":[191],"types":[192],"are":[196,202,214,226,258],"investigated,":[197],"corresponding":[199],"algorithms":[201,225,257],"consequently":[203],"introduced.":[204],"Further,":[205],"top-k":[207],"explored":[215],"deal":[217],"long":[219],"lists.":[222],"tested":[227],"both":[229,284],"synthetic":[230],"real-world":[232],"data.":[233],"Results":[234],"reveal":[235],"estimation":[240],"significantly":[242],"outperforms":[243],"average":[245],"regression":[250],"methods.":[251],"performances":[253],"comparable":[259],"those":[261],"consideration":[267],"results":[271],"investigation":[274],"further":[275],"indicate":[276],"helpful":[281],"inferring":[283],"functions":[286],"degrees":[289],"annotators.":[291]},"counts_by_year":[{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}