{"id":"https://openalex.org/W2105837509","doi":"https://doi.org/10.1145/2505515.2505755","title":"Optimizing plurality for human intelligence tasks","display_name":"Optimizing plurality for human intelligence tasks","publication_year":2013,"publication_date":"2013-10-27","ids":{"openalex":"https://openalex.org/W2105837509","doi":"https://doi.org/10.1145/2505515.2505755","mag":"2105837509"},"language":"en","primary_location":{"id":"doi:10.1145/2505515.2505755","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2505515.2505755","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM international conference on Information & Knowledge Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089642697","display_name":"Luyi Mo","orcid":null},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Luyi Mo","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005847882","display_name":"Reynold Cheng","orcid":"https://orcid.org/0000-0002-9480-9809"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Reynold Cheng","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063695659","display_name":"Ben Kao","orcid":"https://orcid.org/0000-0002-0501-9435"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Ben Kao","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101975561","display_name":"Xuan Yang","orcid":"https://orcid.org/0009-0001-0361-2470"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xuan S. Yang","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021835220","display_name":"Chenghui Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Chenghui Ren","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070712748","display_name":"Siyu Lei","orcid":null},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Siyu Lei","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030458470","display_name":"David W. Cheung","orcid":"https://orcid.org/0000-0002-8343-8440"},"institutions":[{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"David W. Cheung","raw_affiliation_strings":["University of Hong Kong, Hong Kong, Hong Kong"],"affiliations":[{"raw_affiliation_string":"University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I889458895"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011148671","display_name":"Eric Lo","orcid":"https://orcid.org/0000-0003-2679-3945"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Eric Lo","raw_affiliation_strings":["Hong Kong Polytechnic University, Hong Kong, Hong Kong","Hong Kong Polytechnic University , Hong Kong , Hong Kong"],"affiliations":[{"raw_affiliation_string":"Hong Kong Polytechnic University, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I14243506"]},{"raw_affiliation_string":"Hong Kong Polytechnic University , Hong Kong , Hong Kong","institution_ids":["https://openalex.org/I14243506"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5089642697"],"corresponding_institution_ids":["https://openalex.org/I889458895"],"apc_list":null,"apc_paid":null,"fwci":4.8377,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.95116727,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"1929","last_page":"1938"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10326","display_name":"Indoor and Outdoor Localization Technologies","score":0.9846000075340271,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8274224996566772},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.795181393623352},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.6854680776596069},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6395771503448486},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6174713969230652},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.5940366387367249},{"id":"https://openalex.org/keywords/greedy-algorithm","display_name":"Greedy algorithm","score":0.5872659683227539},{"id":"https://openalex.org/keywords/human-intelligence","display_name":"Human intelligence","score":0.4776626527309418},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38344648480415344},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35076943039894104},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.2919456958770752},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11383366584777832},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08895400166511536}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8274224996566772},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.795181393623352},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.6854680776596069},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6395771503448486},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6174713969230652},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.5940366387367249},{"id":"https://openalex.org/C51823790","wikidata":"https://www.wikidata.org/wiki/Q504353","display_name":"Greedy algorithm","level":2,"score":0.5872659683227539},{"id":"https://openalex.org/C105409693","wikidata":"https://www.wikidata.org/wiki/Q5937824","display_name":"Human intelligence","level":2,"score":0.4776626527309418},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38344648480415344},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35076943039894104},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2919456958770752},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11383366584777832},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08895400166511536},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/2505515.2505755","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2505515.2505755","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 22nd ACM international conference on Information & Knowledge Management","raw_type":"proceedings-article"},{"id":"pmh:oai:hub.hku.hk:10722/189633","is_oa":false,"landing_page_url":"http://hdl.handle.net/10722/189633","pdf_url":null,"source":{"id":"https://openalex.org/S4377196271","display_name":"The HKU Scholars Hub (University of Hong Kong)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I889458895","host_organization_name":"University of Hong Kong","host_organization_lineage":["https://openalex.org/I889458895"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference_Paper"},{"id":"pmh:oai:ira.lib.polyu.edu.hk:10397/39866","is_oa":false,"landing_page_url":"http://hdl.handle.net/10397/39866","pdf_url":null,"source":{"id":"https://openalex.org/S4306400205","display_name":"PolyU Institutional Research Archive (Hong Kong Polytechnic University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I14243506","host_organization_name":"Hong Kong Polytechnic University","host_organization_lineage":["https://openalex.org/I14243506"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Paper"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4000000059604645,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1546421159","https://openalex.org/W1551839888","https://openalex.org/W1974554406","https://openalex.org/W1986060983","https://openalex.org/W2028818097","https://openalex.org/W2036523804","https://openalex.org/W2053653724","https://openalex.org/W2056748234","https://openalex.org/W2066671865","https://openalex.org/W2068645020","https://openalex.org/W2076357412","https://openalex.org/W2083293881","https://openalex.org/W2101620066","https://openalex.org/W2121117737","https://openalex.org/W2122067387","https://openalex.org/W2145492473","https://openalex.org/W2168144930","https://openalex.org/W2289836221","https://openalex.org/W2312417930","https://openalex.org/W6663798068","https://openalex.org/W6696926454","https://openalex.org/W6698378874","https://openalex.org/W7049061842"],"related_works":["https://openalex.org/W3183948672","https://openalex.org/W3173606202","https://openalex.org/W3110381201","https://openalex.org/W2948807893","https://openalex.org/W2935909890","https://openalex.org/W2778153218","https://openalex.org/W2758277628","https://openalex.org/W1531601525","https://openalex.org/W1689950805","https://openalex.org/W3131060711"],"abstract_inverted_index":{"In":[0],"a":[1,26,56,66,79,84,113,139,151],"crowdsourcing":[2],"system,":[3],"Human":[4],"Intelligence":[5],"Tasks":[6],"(HITs)":[7],"(e.g.,":[8,166],"translating":[9],"sentences,":[10],"matching":[11],"photos,":[12],"tagging":[13],"videos":[14],"with":[15,154],"keywords)":[16],"can":[17],"be":[18,52,75],"conveniently":[19],"specified.":[20],"HITs":[21,37,49,82,160],"are":[22,32,62,136],"made":[23],"available":[24],"to":[25,64,191],"large":[27],"pool":[28],"of":[29,70,81,92,96,100,144],"workers,":[30],"who":[31],"paid":[33],"upon":[34],"completing":[35],"the":[36,68,71,89,94,105,120,142,145,163,169],"they":[38],"have":[39,44],"selected.":[40],"Since":[41],"workers":[42,61,97],"may":[43,50],"different":[45],"capabilities,":[46],"some":[47],"difficult":[48],"not":[51],"satisfactorily":[53],"performed":[54],"by":[55,138],"single":[57],"worker.":[58],"If":[59],"more":[60],"employed":[63],"perform":[65],"HIT,":[67],"quality":[69,108,143],"HIT's":[72,146],"answer":[73,107,147],"could":[74],"statistically":[76],"improved.":[77],"Given":[78],"set":[80],"and":[83,132,188],"fixed":[85],"\"budget\",":[86],"we":[87],"address":[88],"important":[90],"problem":[91,123],"determining":[93],"number":[95],"(or":[98],"plurality)":[99],"each":[101],"HIT":[102,140],"so":[103],"that":[104,161,198],"overall":[106],"is":[109,171],"optimized.":[110],"We":[111,125,157,173,182],"propose":[112,174],"dynamic":[114],"programming":[115],"(DP)":[116],"algorithm":[117,178,201],"for":[118,159,179],"solving":[119],"plurality":[121],"assignment":[122],"(PAP).":[124],"identify":[126],"two":[127,164],"interesting":[128],"properties,":[129],"namely,":[130],"monotonicity":[131],"diminishing":[133],"return,":[134],"which":[135],"satisfied":[137],"if":[141],"increases":[148],"monotonically":[149],"at":[150],"decreasing":[152],"rate":[153],"its":[155],"plurality.":[156],"show":[158,197],"satisfy":[162],"properties":[165],"multiple-choice-question":[167],"HITs),":[168],"PAP":[170],"approximable.":[172],"an":[175],"efficient":[176],"greedy":[177,200],"such":[180],"case.":[181],"conduct":[183],"extensive":[184],"experiments":[185,196],"on":[186],"synthetic":[187],"real":[189],"datasets":[190],"evaluate":[192],"our":[193,199],"algorithms.":[194],"Our":[195],"provides":[202],"close-to-optimal":[203],"solutions":[204],"in":[205],"practice.":[206]},"counts_by_year":[{"year":2022,"cited_by_count":2},{"year":2019,"cited_by_count":1},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":4}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}