{"id":"https://openalex.org/W2258849743","doi":"https://doi.org/10.1145/2835776.2835797","title":"Crowdsourcing High Quality Labels with a Tight Budget","display_name":"Crowdsourcing High Quality Labels with a Tight Budget","publication_year":2016,"publication_date":"2016-02-04","ids":{"openalex":"https://openalex.org/W2258849743","doi":"https://doi.org/10.1145/2835776.2835797","mag":"2258849743"},"language":"en","primary_location":{"id":"doi:10.1145/2835776.2835797","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2835776.2835797","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2835797&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"http://dl.acm.org/ft_gateway.cfm?id=2835797&type=pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101936139","display_name":"Qi Li","orcid":"https://orcid.org/0000-0002-2825-3531"},"institutions":[{"id":"https://openalex.org/I115441956","display_name":"Buffalo State University","ror":"https://ror.org/05ms04m92","country_code":"US","type":"education","lineage":["https://openalex.org/I115441956"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Qi Li","raw_affiliation_strings":["SUNY Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SUNY Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I115441956","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001030192","display_name":"Fenglong Ma","orcid":"https://orcid.org/0000-0002-4999-0303"},"institutions":[{"id":"https://openalex.org/I115441956","display_name":"Buffalo State University","ror":"https://ror.org/05ms04m92","country_code":"US","type":"education","lineage":["https://openalex.org/I115441956"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Fenglong Ma","raw_affiliation_strings":["SUNY Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SUNY Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I115441956","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100781388","display_name":"Jing Gao","orcid":"https://orcid.org/0000-0002-4461-7710"},"institutions":[{"id":"https://openalex.org/I115441956","display_name":"Buffalo State University","ror":"https://ror.org/05ms04m92","country_code":"US","type":"education","lineage":["https://openalex.org/I115441956"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jing Gao","raw_affiliation_strings":["SUNY Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SUNY Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I115441956","https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100732938","display_name":"L\u00fc Su","orcid":"https://orcid.org/0000-0001-7223-543X"},"institutions":[{"id":"https://openalex.org/I115441956","display_name":"Buffalo State University","ror":"https://ror.org/05ms04m92","country_code":"US","type":"education","lineage":["https://openalex.org/I115441956"]},{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lu Su","raw_affiliation_strings":["SUNY Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SUNY Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I115441956","https://openalex.org/I63190737"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038627779","display_name":"Christopher J. Quinn","orcid":"https://orcid.org/0000-0002-9053-1504"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher J. Quinn","raw_affiliation_strings":["Purdue University, West Lafayette, IN, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":57,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"237","last_page":"246"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8549677729606628},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8343243598937988},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.7106504440307617},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6926379203796387},{"id":"https://openalex.org/keywords/budget-constraint","display_name":"Budget constraint","score":0.6286576390266418},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6101488471031189},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5042577981948853},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.49399644136428833},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4154684841632843},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.299903929233551},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.07262733578681946}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8549677729606628},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8343243598937988},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.7106504440307617},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6926379203796387},{"id":"https://openalex.org/C8505890","wikidata":"https://www.wikidata.org/wiki/Q605095","display_name":"Budget constraint","level":2,"score":0.6286576390266418},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6101488471031189},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5042577981948853},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.49399644136428833},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4154684841632843},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.299903929233551},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.07262733578681946},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C133425853","wikidata":"https://www.wikidata.org/wiki/Q60571","display_name":"Neoclassical economics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2835776.2835797","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2835776.2835797","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2835797&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/2835776.2835797","is_oa":true,"landing_page_url":"https://doi.org/10.1145/2835776.2835797","pdf_url":"http://dl.acm.org/ft_gateway.cfm?id=2835797&type=pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5199999809265137}],"awards":[{"id":"https://openalex.org/G4761247364","display_name":"III: Small: Collaborative Research: Conflicts to Harmony: Integrating Massive Data by Trustworthiness Estimation and Truth Discovery","funder_award_id":"1319973","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2258849743.pdf","grobid_xml":"https://content.openalex.org/works/W2258849743.grobid-xml"},"referenced_works_count":42,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W108763474","https://openalex.org/W118259468","https://openalex.org/W1541505110","https://openalex.org/W1601808502","https://openalex.org/W1633675443","https://openalex.org/W1703951723","https://openalex.org/W1858303725","https://openalex.org/W1970381522","https://openalex.org/W1992766323","https://openalex.org/W2013103455","https://openalex.org/W2033314834","https://openalex.org/W2034771068","https://openalex.org/W2046406837","https://openalex.org/W2046711200","https://openalex.org/W2055733113","https://openalex.org/W2064677871","https://openalex.org/W2067708565","https://openalex.org/W2086413055","https://openalex.org/W2105268242","https://openalex.org/W2116664070","https://openalex.org/W2119567691","https://openalex.org/W2120826095","https://openalex.org/W2121200917","https://openalex.org/W2125943921","https://openalex.org/W2134305421","https://openalex.org/W2136150876","https://openalex.org/W2141282920","https://openalex.org/W2142518823","https://openalex.org/W2146928171","https://openalex.org/W2149273804","https://openalex.org/W2149906572","https://openalex.org/W2152009989","https://openalex.org/W2162815002","https://openalex.org/W2163522723","https://openalex.org/W2164124780","https://openalex.org/W2182472765","https://openalex.org/W2251049298","https://openalex.org/W2290431464","https://openalex.org/W2408493841","https://openalex.org/W2623293810","https://openalex.org/W3021909058"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W1503094549","https://openalex.org/W4384486036","https://openalex.org/W135177976","https://openalex.org/W2337920774","https://openalex.org/W4318823662","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W3207526114","https://openalex.org/W4286908577"],"abstract_inverted_index":{"In":[0],"the":[1,9,47,53,56,128,133,140,171,177,182,186,200,221,225],"past":[2],"decade,":[3],"commercial":[4],"crowdsourcing":[5,210],"platforms":[6],"have":[7,98],"revolutionized":[8],"ways":[10],"of":[11,49,55,72,92,135],"classifying":[12],"and":[13,52,131,158,238],"annotating":[14],"data,":[15],"especially":[16],"for":[17,22,30,103,170,185],"large":[18,31],"datasets.":[19],"Obtaining":[20],"labels":[21],"a":[23,76,112,144,154,159,215],"single":[24],"instance":[25,172],"can":[26,63,180,202],"be":[27,85,203],"inexpensive,":[28],"but":[29,67,91],"datasets,":[32],"it":[33],"is":[34,151,163,191,223],"important":[35],"to":[36,87,122,173,194],"allocate":[37],"budgets":[38],"wisely.":[39],"With":[40],"limited":[41],"budgets,":[42],"requesters":[43,83,121],"must":[44],"trade-off":[45],"between":[46],"quantity":[48,66,237],"labeled":[50,136],"instances":[51,74,90,137],"quality":[54,71,102,130,141,239],"final":[57],"results.":[58],"Existing":[59],"budget":[60,114,148,201,222,232],"allocation":[61,115,149,233],"methods":[62,234],"achieve":[64,139],"good":[65],"cannot":[68],"guarantee":[69],"high":[70],"individual":[73],"under":[75,143],"tight":[77,145],"budget.":[78,146],"However,":[79],"in":[80],"some":[81],"scenarios,":[82],"may":[84,97],"willing":[86],"label":[88],"fewer":[89],"higher":[93],"quality.":[94],"Moreover,":[95],"they":[96],"different":[99,104],"requirements":[100,126],"on":[101,127,207],"tasks.":[105],"To":[106],"address":[107],"these":[108],"challenges,":[109],"we":[110],"propose":[111],"flexible":[113],"framework":[116,190,228],"called":[117],"Requallo.":[118],"Requallo":[119,189,227],"allows":[120],"set":[123],"their":[124],"specific":[125],"labeling":[129,161],"maximizes":[132],"number":[134],"that":[138,179,199,219],"requirement":[142],"The":[147,165,188],"problem":[150],"modeled":[152],"as":[153,176,212,214],"Markov":[155],"decision":[156],"process":[157],"sequential":[160],"policy":[162,167],"produced.":[164],"proposed":[166,226],"greedily":[168],"searches":[169],"query":[174],"next":[175],"one":[178],"provide":[181],"maximum":[183],"reward":[184],"goal.":[187],"further":[192],"extended":[193],"consider":[195],"worker":[196],"reliability":[197],"so":[198],"better":[204],"allocated.":[205],"Experiments":[206],"two":[208],"real-world":[209],"tasks":[211],"well":[213],"simulated":[216],"task":[217],"demonstrate":[218],"when":[220],"tight,":[224],"outperforms":[229],"existing":[230],"state-of-the-art":[231],"from":[235],"both":[236],"aspects.":[240]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":12},{"year":2018,"cited_by_count":15},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}