{"id":"https://openalex.org/W1975787960","doi":"https://doi.org/10.1145/2009916.2010129","title":"Learning to rank from a noisy crowd","display_name":"Learning to rank from a noisy crowd","publication_year":2011,"publication_date":"2011-07-24","ids":{"openalex":"https://openalex.org/W1975787960","doi":"https://doi.org/10.1145/2009916.2010129","mag":"1975787960"},"language":"en","primary_location":{"id":"doi:10.1145/2009916.2010129","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2009916.2010129","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046535830","display_name":"Abhimanu Kumar","orcid":"https://orcid.org/0000-0003-4529-7883"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Abhimanu Kumar","raw_affiliation_strings":["University Of Texas at Austin, Austin, TX, USA","University of Texas at Austin, Austin, TX USA"],"affiliations":[{"raw_affiliation_string":"University Of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, TX USA","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023397430","display_name":"Matthew Lease","orcid":"https://orcid.org/0000-0002-0056-2834"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matthew Lease","raw_affiliation_strings":["University Of Texas at Austin, Austin, TX, USA","University of Texas at Austin, Austin, TX USA"],"affiliations":[{"raw_affiliation_string":"University Of Texas at Austin, Austin, TX, USA","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin, Austin, TX USA","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5046535830"],"corresponding_institution_ids":["https://openalex.org/I86519309"],"apc_list":null,"apc_paid":null,"fwci":5.43013915,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.92811535,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"1221","last_page":"1222"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.8213698267936707},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7369604110717773},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.7327207326889038},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7277112007141113},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6668708324432373},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6567928194999695},{"id":"https://openalex.org/keywords/binary-classification","display_name":"Binary classification","score":0.5826037526130676},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5798953175544739},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5761258602142334},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.5649279356002808},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.528392493724823},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.48073434829711914},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4623950719833374},{"id":"https://openalex.org/keywords/binary-number","display_name":"Binary number","score":0.4622376561164856},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3560560345649719},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16850990056991577},{"id":"https://openalex.org/keywords/support-vector-machine","display_name":"Support vector machine","score":0.10189920663833618}],"concepts":[{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.8213698267936707},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7369604110717773},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.7327207326889038},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7277112007141113},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6668708324432373},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6567928194999695},{"id":"https://openalex.org/C66905080","wikidata":"https://www.wikidata.org/wiki/Q17005494","display_name":"Binary classification","level":3,"score":0.5826037526130676},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5798953175544739},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5761258602142334},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.5649279356002808},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.528392493724823},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.48073434829711914},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4623950719833374},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.4622376561164856},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3560560345649719},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16850990056991577},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.10189920663833618},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2009916.2010129","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2009916.2010129","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 34th international ACM SIGIR conference on Research and development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.4000000059604645}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W1970381522","https://openalex.org/W1975787960","https://openalex.org/W2055601365","https://openalex.org/W2108862644","https://openalex.org/W2125943921","https://openalex.org/W2129245267"],"related_works":["https://openalex.org/W3127142483","https://openalex.org/W4385565564","https://openalex.org/W2898073868","https://openalex.org/W2138488530","https://openalex.org/W2971071571","https://openalex.org/W2798835721","https://openalex.org/W2387658907","https://openalex.org/W2922169395","https://openalex.org/W25098770","https://openalex.org/W2385796165"],"abstract_inverted_index":{"We":[0,14,85],"study":[1],"how":[2],"to":[3,10,69,82],"best":[4],"use":[5],"crowdsourced":[6],"relevance":[7,33],"judgments":[8,34],"learning":[9,92],"rank":[11],"[1,":[12],"7].":[13],"integrate":[15],"two":[16],"lines":[17],"of":[18,45,90],"prior":[19,70],"work:":[20],"unreliable":[21],"crowd-based":[22],"binary":[23,26],"annotation":[24,50],"for":[25,38,106],"classification":[27],"[5,":[28],"3],":[29],"and":[30,55],"aggregating":[31],"graded":[32],"from":[35],"reliable":[36],"experts":[37],"ranking":[39],"[7].":[40],"To":[41],"model":[42],"varying":[43,53],"performance":[44],"the":[46,91,103],"crowd,":[47],"we":[48],"simulate":[49],"noise":[51,95],"with":[52,102],"magnitude":[54],"distributional":[56],"properties.":[57],"Evaluation":[58],"on":[59],"three":[60],"LETOR":[61],"test":[62],"collections":[63],"reveals":[64],"a":[65],"striking":[66],"trend":[67],"contrary":[68],"studies:":[71],"single":[72],"labeling":[73],"outperforms":[74],"consensus":[75],"methods":[76],"in":[77],"maximizing":[78],"learner":[79],"accuracy":[80],"relative":[81],"annotator":[83],"e\u00fdort.":[84],"also":[86],"see":[87],"surprising":[88],"consistency":[89],"curve":[93],"across":[94],"distributions,":[96],"as":[97,99],"well":[98],"greater":[100],"challenge":[101],"adversarial":[104],"case":[105],"multi-class":[107],"labeling.":[108]},"counts_by_year":[{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":2},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2013,"cited_by_count":2},{"year":2012,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}