{"id":"https://openalex.org/W2158751658","doi":"https://doi.org/10.1145/2465529.2465761","title":"Efficient crowdsourcing for multi-class labeling","display_name":"Efficient crowdsourcing for multi-class labeling","publication_year":2013,"publication_date":"2013-06-17","ids":{"openalex":"https://openalex.org/W2158751658","doi":"https://doi.org/10.1145/2465529.2465761","mag":"2158751658"},"language":"en","primary_location":{"id":"doi:10.1145/2465529.2465761","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2465529.2465761","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGMETRICS/international conference on Measurement and modeling of computer systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028448267","display_name":"David R. Karger","orcid":"https://orcid.org/0000-0002-0024-5847"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David R. Karger","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028243041","display_name":"Sewoong Oh","orcid":"https://orcid.org/0000-0002-8975-8306"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sewoong Oh","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029499294","display_name":"Devavrat Shah","orcid":"https://orcid.org/0000-0003-0737-3259"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devavrat Shah","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":103,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"81","last_page":"92"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12879","display_name":"Distributed Sensor Networks and Detection Algorithms","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8296505212783813},{"id":"https://openalex.org/keywords/redundancy","display_name":"Redundancy (engineering)","score":0.6887563467025757},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.658414900302887},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.5813780426979065},{"id":"https://openalex.org/keywords/triple-modular-redundancy","display_name":"Triple modular redundancy","score":0.5085409879684448},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5025899410247803},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.46120685338974},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4476706385612488},{"id":"https://openalex.org/keywords/majority-rule","display_name":"Majority rule","score":0.4354647099971771},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4235779345035553},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.33045390248298645}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8296505212783813},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.6887563467025757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.658414900302887},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.5813780426979065},{"id":"https://openalex.org/C196371267","wikidata":"https://www.wikidata.org/wiki/Q3998979","display_name":"Triple modular redundancy","level":3,"score":0.5085409879684448},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5025899410247803},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46120685338974},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4476706385612488},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.4354647099971771},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4235779345035553},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33045390248298645},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2465529.2465761","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2465529.2465761","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM SIGMETRICS/international conference on Measurement and modeling of computer systems","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320333591","display_name":"Multidisciplinary University Research Initiative","ror":null},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W390146837","https://openalex.org/W1536930200","https://openalex.org/W1988560072","https://openalex.org/W2000157792","https://openalex.org/W2006544720","https://openalex.org/W2021145763","https://openalex.org/W2026709222","https://openalex.org/W2036625304","https://openalex.org/W2049633694","https://openalex.org/W2098865355","https://openalex.org/W2119830539","https://openalex.org/W2125943921","https://openalex.org/W2127008633","https://openalex.org/W2129345386","https://openalex.org/W2134305421","https://openalex.org/W2140890285","https://openalex.org/W2142518823","https://openalex.org/W2144372981","https://openalex.org/W2144730813","https://openalex.org/W2149273804","https://openalex.org/W2163522723","https://openalex.org/W2554839354","https://openalex.org/W2596585349","https://openalex.org/W2597289420","https://openalex.org/W2604272474","https://openalex.org/W2905110430","https://openalex.org/W2999905431","https://openalex.org/W4299379997","https://openalex.org/W4387723197","https://openalex.org/W6679158198","https://openalex.org/W6679959949","https://openalex.org/W6681102160","https://openalex.org/W6682171051"],"related_works":["https://openalex.org/W58658798","https://openalex.org/W2039876276","https://openalex.org/W3114375939","https://openalex.org/W3008821054","https://openalex.org/W2797678940","https://openalex.org/W2759696718","https://openalex.org/W2359816675","https://openalex.org/W2605569989","https://openalex.org/W4210531477","https://openalex.org/W2603119174"],"abstract_inverted_index":{"Crowdsourcing":[0],"systems":[1,99],"like":[2],"Amazon's":[3],"Mechanical":[4],"Turk":[5],"have":[6,154],"emerged":[7],"as":[8,20,139,141],"an":[9,130],"effective":[10],"large-scale":[11],"human-powered":[12],"platform":[13],"for":[14,94,97],"performing":[15],"tasks":[16,38],"in":[17,215],"domains":[18],"such":[19],"image":[21],"classification,":[22],"data":[23],"entry,":[24],"recommendation,":[25],"and":[26,37,47,83,100,200,221],"proofreading.":[27],"Since":[28],"workers":[29,220],"are":[30,40,45],"low-paid":[31],"(a":[32],"few":[33],"cents":[34],"per":[35,144],"task)":[36],"performed":[39],"monotonous,":[41],"the":[42,77,102,106,142,157,165,179,195,210,216,229],"answers":[43,160],"obtained":[44],"noisy":[46,95],"hence":[48],"unreliable.":[49],"To":[50],"obtain":[51,76,129],"reliable":[52],"estimates,":[53],"it":[54,125],"is":[55,74,126,146,164,169,178],"essential":[56],"to":[57,75,115,128,132],"utilize":[58],"appropriate":[59],"inference":[60],"algorithms":[61],"(e.g.":[62],"Majority":[63],"voting)":[64],"coupled":[65],"with":[66,136],"structured":[67],"redundancy":[68,143,199],"through":[69,171],"task":[70,134,145,152],"assignment.":[71],"Our":[72],"goal":[73],"best":[78,180],"possible":[79,127,181],"trade-off":[80,183,197],"between":[81,198],"reliability":[82,201],"redundancy.":[84],"In":[85],"this":[86,177],"paper,":[87],"we":[88,122,208],"consider":[89],"a":[90,117,172,190,223],"general":[91],"probabilistic":[92,173],"model":[93],"observations":[96],"crowd-sourcing":[98],"pose":[101],"problem":[103],"of":[104,156,194,212,218],"minimizing":[105],"total":[107],"price":[108],"(i.e.":[109],"redundancy)":[110],"that":[111,124,168],"must":[112],"be":[113],"paid":[114],"achieve":[116],"target":[118],"overall":[119],"reliability.":[120],"Concretely,":[121],"show":[123],"answer":[131],"each":[133,151],"correctly":[135],"probability":[137],"1-\u03b5":[138],"long":[140],"O((K/q)":[147],"log":[148],"(K/\u03b5)),":[149],"where":[150],"can":[153,187],"any":[155,184],"$K$":[158],"distinct":[159],"equally":[161],"likely,":[162],"q":[163],"crowd-quality":[166],"parameter":[167],"defined":[170],"model.":[174],"Further,":[175,207],"effectively":[176],"redundancy-accuracy":[182,230],"system":[185],"design":[186],"achieve.":[188],"Such":[189],"single-parameter":[191],"crisp":[192],"characterization":[193],"(order-)optimal":[196],"has":[202],"various":[203],"useful":[204],"operational":[205],"consequences.":[206],"analyze":[209],"robustness":[211],"our":[213],"approach":[214],"presence":[217],"adversarial":[219],"provide":[222],"bound":[224],"on":[225,228],"their":[226],"influence":[227],"trade-off.":[231]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":11},{"year":2019,"cited_by_count":10},{"year":2018,"cited_by_count":9},{"year":2017,"cited_by_count":10},{"year":2016,"cited_by_count":15},{"year":2015,"cited_by_count":16},{"year":2014,"cited_by_count":13},{"year":2013,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}