{"id":"https://openalex.org/W2170493558","doi":"https://doi.org/10.1145/2736277.2741102","title":"Incentivizing High Quality Crowdwork","display_name":"Incentivizing High Quality Crowdwork","publication_year":2015,"publication_date":"2015-05-18","ids":{"openalex":"https://openalex.org/W2170493558","doi":"https://doi.org/10.1145/2736277.2741102","mag":"2170493558"},"language":"en","primary_location":{"id":"doi:10.1145/2736277.2741102","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2736277.2741102","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1503.05897","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101621594","display_name":"Chien-Ju Ho","orcid":"https://orcid.org/0000-0002-7558-7702"},"institutions":[{"id":"https://openalex.org/I161318765","display_name":"University of California, Los Angeles","ror":"https://ror.org/046rm7j60","country_code":"US","type":"education","lineage":["https://openalex.org/I161318765"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chien-Ju Ho","raw_affiliation_strings":["University of California, Los Angeles, Los Angeles, USA"],"affiliations":[{"raw_affiliation_string":"University of California, Los Angeles, Los Angeles, USA","institution_ids":["https://openalex.org/I161318765"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058550942","display_name":"Aleksandrs Slivkins","orcid":"https://orcid.org/0000-0001-6899-6383"},"institutions":[{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Aleksandrs Slivkins","raw_affiliation_strings":["Microsoft Research, New York City, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, New York City, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051257652","display_name":"Siddharth Suri","orcid":"https://orcid.org/0000-0002-1318-8140"},"institutions":[{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]},{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siddharth Suri","raw_affiliation_strings":["Microsoft Research, New York City, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, New York City, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043117896","display_name":"Jennifer Wortman Vaughan","orcid":"https://orcid.org/0000-0002-7807-2018"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":null,"type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jennifer Wortman Vaughan","raw_affiliation_strings":["Microsoft Research, New York City, USA"],"affiliations":[{"raw_affiliation_string":"Microsoft Research, New York City, USA","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5101621594"],"corresponding_institution_ids":["https://openalex.org/I161318765"],"apc_list":null,"apc_paid":null,"fwci":27.1611,"has_fulltext":true,"cited_by_count":140,"citation_normalized_percentile":{"value":0.99444973,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"419","last_page":"429"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11182","display_name":"Auction Theory and Applications","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10646","display_name":"Experimental Behavioral Economics Studies","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/payment","display_name":"Payment","score":0.7938041687011719},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7523256540298462},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7508240938186646},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.7293490171432495},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.7205777764320374},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6759219169616699},{"id":"https://openalex.org/keywords/salient","display_name":"Salient","score":0.5264414548873901},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4893478751182556},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3276481628417969},{"id":"https://openalex.org/keywords/risk-analysis","display_name":"Risk analysis (engineering)","score":0.32558923959732056},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.15152427554130554},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.14641684293746948},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.10844027996063232},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09604591131210327}],"concepts":[{"id":"https://openalex.org/C145097563","wikidata":"https://www.wikidata.org/wiki/Q1148747","display_name":"Payment","level":2,"score":0.7938041687011719},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7523256540298462},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7508240938186646},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.7293490171432495},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.7205777764320374},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6759219169616699},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.5264414548873901},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4893478751182556},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3276481628417969},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.32558923959732056},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.15152427554130554},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.14641684293746948},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.10844027996063232},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09604591131210327},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/2736277.2741102","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2736277.2741102","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th International Conference on World Wide Web","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1503.05897","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1503.05897","pdf_url":"https://arxiv.org/pdf/1503.05897","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.695.7229","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.695.7229","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.www2015.it/documents/proceedings/proceedings/p419.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.698.8650","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.698.8650","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.jennwv.com/papers/pbp.pdf","raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1503.05897","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1503.05897","pdf_url":"https://arxiv.org/pdf/1503.05897","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.4099999964237213,"id":"https://metadata.un.org/sdg/1","display_name":"No poverty"}],"awards":[{"id":"https://openalex.org/G443259337","display_name":"CAREER: Learning- and Incentives-Based Techniques for Aggregating Community-Generated Data","funder_award_id":"1054911","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2170493558.pdf","grobid_xml":"https://content.openalex.org/works/W2170493558.grobid-xml"},"referenced_works_count":55,"referenced_works":["https://openalex.org/W566586761","https://openalex.org/W602637153","https://openalex.org/W1497882404","https://openalex.org/W1509039412","https://openalex.org/W1523117923","https://openalex.org/W1589928278","https://openalex.org/W1797268635","https://openalex.org/W1820894600","https://openalex.org/W1975450585","https://openalex.org/W1979430517","https://openalex.org/W1987953153","https://openalex.org/W1988240765","https://openalex.org/W2005917549","https://openalex.org/W2022550861","https://openalex.org/W2041411104","https://openalex.org/W2049297495","https://openalex.org/W2073532071","https://openalex.org/W2098865355","https://openalex.org/W2099044475","https://openalex.org/W2105705711","https://openalex.org/W2107394329","https://openalex.org/W2122028729","https://openalex.org/W2125943921","https://openalex.org/W2129345386","https://openalex.org/W2131608330","https://openalex.org/W2140890285","https://openalex.org/W2141708418","https://openalex.org/W2151401338","https://openalex.org/W2152928267","https://openalex.org/W2164976104","https://openalex.org/W2168037022","https://openalex.org/W2168112513","https://openalex.org/W2169047435","https://openalex.org/W2182992846","https://openalex.org/W2187912528","https://openalex.org/W2398690976","https://openalex.org/W2735460460","https://openalex.org/W2951733600","https://openalex.org/W3121522380","https://openalex.org/W3122276354","https://openalex.org/W3123305365","https://openalex.org/W3123895079","https://openalex.org/W3124878131","https://openalex.org/W3125780738","https://openalex.org/W3126123353","https://openalex.org/W3181687612","https://openalex.org/W4241226388","https://openalex.org/W4244837625","https://openalex.org/W4248300944","https://openalex.org/W4248915016","https://openalex.org/W4253356459","https://openalex.org/W4253633548","https://openalex.org/W4254376918","https://openalex.org/W4312713777","https://openalex.org/W4313047369"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2511805441","https://openalex.org/W2108936692","https://openalex.org/W4317927411","https://openalex.org/W2603064225"],"abstract_inverted_index":{"We":[0,13,29,71,118,159],"study":[1],"the":[2,9,37,45,58,63,85,94,101,104,107,115,140,145,216,237],"causal":[3],"effects":[4],"of":[5,11,47,57,74,87,106,122,211,231,239,260],"financial":[6],"incentives":[7,261],"on":[8,15,36,181],"quality":[10,27,97,153],"crowdwork.":[12],"focus":[14],"performance-based":[16,189],"payments":[17,20,180],"(PBPs),":[18],"bonus":[19,108],"awarded":[21],"to":[22,93,99,113,137,150,165,185,222],"workers":[23,149,192],"for":[24,76,96,124,135,257],"producing":[25],"high":[26],"work.":[28],"design":[30],"and":[31,51,62,234],"run":[32],"randomized":[33],"behavioral":[34],"experiments":[35,176],"popular":[38],"crowdsourcing":[39,263],"platform":[40],"Amazon":[41],"Mechanical":[42,182],"Turk":[43,183],"with":[44,245],"goal":[46],"understanding":[48],"when,":[49],"where,":[50],"why":[52],"PBPs":[53,78,88,126,136],"help,":[54],"identifying":[55],"properties":[56],"payment,":[59],"payment":[60],"structure,":[61],"task":[64,141,146,169],"itself":[65],"that":[66,134,178,191,214,236],"make":[67,114],"them":[68],"most":[69],"effective.":[70],"provide":[72],"examples":[73,121],"tasks":[75,123],"which":[77,125],"do":[79,127],"improve":[80,129,138],"quality.":[81,130],"For":[82],"such":[83],"tasks,":[84],"effectiveness":[86],"is":[89,170,202],"not":[90,128],"too":[91],"sensitive":[92],"threshold":[95],"required":[98],"receive":[100],"bonus,":[102],"while":[103],"magnitude":[105],"must":[109,142,147],"be":[110,143,197,252],"large":[111],"enough":[112],"reward":[116],"salient.":[117],"also":[119,160],"present":[120],"Our":[131],"results":[132],"suggest":[133,177],"quality,":[139],"effort-responsive:":[144],"allow":[148],"produce":[151],"higher":[152],"work":[154,195],"by":[155],"exerting":[156],"more":[157],"effort.":[158],"give":[161],"a":[162,168,172,208,224,255],"simple":[163],"method":[164],"determine":[166],"if":[167,199],"effort-responsive":[171],"priori.":[173],"Furthermore,":[174],"our":[175,246],"all":[179],"are,":[184],"some":[186],"degree,":[187],"implicitly":[188],"in":[190,243,262],"believe":[193],"their":[194,200],"may":[196,251],"rejected":[198],"performance":[201],"sufficiently":[203],"poor.":[204],"Finally,":[205],"we":[206],"propose":[207],"new":[209],"model":[210,219,241,250],"worker":[212],"behavior":[213],"extends":[215],"standard":[217],"principal-agent":[218],"from":[220],"economics":[221],"include":[223],"worker's":[225],"subjective":[226],"beliefs":[227],"about":[228],"his":[229],"likelihood":[230],"being":[232],"paid,":[233],"show":[235],"predictions":[238],"this":[240],"are":[242],"line":[244],"experimental":[247],"findings.":[248],"This":[249],"useful":[253],"as":[254],"foundation":[256],"theoretical":[258],"studies":[259],"markets.":[264]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":21},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":21},{"year":2018,"cited_by_count":15},{"year":2017,"cited_by_count":15},{"year":2016,"cited_by_count":15},{"year":2015,"cited_by_count":8},{"year":2014,"cited_by_count":2}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}