{"id":"https://openalex.org/W7155892172","doi":"https://doi.org/10.1145/3774904.3792170","title":"The Power of Penalties: Negativity-Aware Incentives for High-Quality Crowdsourced Data Labeling","display_name":"The Power of Penalties: Negativity-Aware Incentives for High-Quality Crowdsourced Data Labeling","publication_year":2026,"publication_date":"2026-04-12","ids":{"openalex":"https://openalex.org/W7155892172","doi":"https://doi.org/10.1145/3774904.3792170"},"language":null,"primary_location":{"id":"doi:10.1145/3774904.3792170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3774904.3792170","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3774904.3792170","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134721581","display_name":"Kai Wang","orcid":"https://orcid.org/0009-0002-3768-6088"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Wang","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0009-0002-3768-6088","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069512988","display_name":"Runze Wu","orcid":"https://orcid.org/0000-0002-6986-5825"},"institutions":[{"id":"https://openalex.org/I4210091137","display_name":"NetEase (China)","ror":"https://ror.org/00fp6fj05","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091137"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Runze Wu","raw_affiliation_strings":["NetEase Fuxi AI Lab, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-6986-5825","affiliations":[{"raw_affiliation_string":"NetEase Fuxi AI Lab, Hangzhou, China","institution_ids":["https://openalex.org/I4210091137"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030275295","display_name":"Yu Xiong","orcid":"https://orcid.org/0000-0002-9656-6193"},"institutions":[{"id":"https://openalex.org/I4210091137","display_name":"NetEase (China)","ror":"https://ror.org/00fp6fj05","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091137"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Xiong","raw_affiliation_strings":["NetEase Fuxi AI Lab, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-9656-6193","affiliations":[{"raw_affiliation_string":"NetEase Fuxi AI Lab, Hangzhou, China","institution_ids":["https://openalex.org/I4210091137"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008194128","display_name":"Haifeng Sun","orcid":"https://orcid.org/0000-0003-3072-7422"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haifeng Sun","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0009-0008-4980-6263","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000173131","display_name":"Anran Li","orcid":"https://orcid.org/0000-0002-3592-4153"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Anran Li","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-3592-4153","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050393292","display_name":"Shaojie Tang","orcid":"https://orcid.org/0000-0001-9261-5210"},"institutions":[{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shaojie Tang","raw_affiliation_strings":["University at Buffalo, Buffalo, New York, USA"],"raw_orcid":"https://orcid.org/0000-0001-9261-5210","affiliations":[{"raw_affiliation_string":"University at Buffalo, Buffalo, New York, USA","institution_ids":["https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134681548","display_name":"Changjie Fan","orcid":"https://orcid.org/0000-0001-5420-0516"},"institutions":[{"id":"https://openalex.org/I4210091137","display_name":"NetEase (China)","ror":"https://ror.org/00fp6fj05","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210091137"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Changjie Fan","raw_affiliation_strings":["NetEase Fuxi AI Lab, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-5420-0516","affiliations":[{"raw_affiliation_string":"NetEase Fuxi AI Lab, Hangzhou, China","institution_ids":["https://openalex.org/I4210091137"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100341802","display_name":"Xiang\u2010Yang Li","orcid":"https://orcid.org/0000-0002-6070-6625"},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang-Yang Li","raw_affiliation_strings":["University of Science and Technology of China, Hefei, China"],"raw_orcid":"https://orcid.org/0000-0002-6070-6625","affiliations":[{"raw_affiliation_string":"University of Science and Technology of China, Hefei, China","institution_ids":["https://openalex.org/I126520041"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"51","last_page":"62"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9908999800682068,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.00419999985024333,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9229000210762024},{"id":"https://openalex.org/keywords/incentive","display_name":"Incentive","score":0.6851999759674072},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5055000185966492},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4932999908924103},{"id":"https://openalex.org/keywords/valuation","display_name":"Valuation (finance)","score":0.4546999931335449},{"id":"https://openalex.org/keywords/shapley-value","display_name":"Shapley value","score":0.44589999318122864},{"id":"https://openalex.org/keywords/mechanism-design","display_name":"Mechanism design","score":0.4449999928474426},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4316999912261963}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9229000210762024},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7024999856948853},{"id":"https://openalex.org/C29122968","wikidata":"https://www.wikidata.org/wiki/Q1414816","display_name":"Incentive","level":2,"score":0.6851999759674072},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5055000185966492},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4932999908924103},{"id":"https://openalex.org/C186027771","wikidata":"https://www.wikidata.org/wiki/Q4008379","display_name":"Valuation (finance)","level":2,"score":0.4546999931335449},{"id":"https://openalex.org/C199022921","wikidata":"https://www.wikidata.org/wiki/Q240046","display_name":"Shapley value","level":3,"score":0.44589999318122864},{"id":"https://openalex.org/C153517567","wikidata":"https://www.wikidata.org/wiki/Q26090","display_name":"Mechanism design","level":2,"score":0.4449999928474426},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.44269999861717224},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4316999912261963},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.42590001225471497},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.41130000352859497},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.4041000008583069},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3869999945163727},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.38420000672340393},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.3499999940395355},{"id":"https://openalex.org/C2776145971","wikidata":"https://www.wikidata.org/wiki/Q30673951","display_name":"Labeled data","level":2,"score":0.34119999408721924},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3021000027656555},{"id":"https://openalex.org/C152124472","wikidata":"https://www.wikidata.org/wiki/Q1204361","display_name":"Redundancy (engineering)","level":2,"score":0.29649999737739563},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.29010000824928284},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2775000035762787},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2624000012874603}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3774904.3792170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3774904.3792170","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3774904.3792170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3774904.3792170","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM Web Conference 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7001476287841797,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W118259468","https://openalex.org/W2170493558","https://openalex.org/W2487898712","https://openalex.org/W2585226541","https://openalex.org/W2963319311","https://openalex.org/W3006251828","https://openalex.org/W3163332892","https://openalex.org/W4225091060","https://openalex.org/W4312892559","https://openalex.org/W4367047193","https://openalex.org/W4382365422","https://openalex.org/W4382681553","https://openalex.org/W4385343670","https://openalex.org/W4386065397","https://openalex.org/W4396218396","https://openalex.org/W4396723257","https://openalex.org/W4396757559","https://openalex.org/W4401507982"],"related_works":[],"abstract_inverted_index":{"High-quality":[0],"data":[1,189],"labeling":[2,116,156],"is":[3],"essential":[4],"for":[5,158],"training":[6],"robust":[7],"machine":[8],"learning":[9],"models;":[10],"however,":[11],"existing":[12],"methods":[13],"often":[14],"ignore":[15],"fraud":[16,166],"or":[17],"assume":[18],"non-negative":[19],"worker":[20],"utility,":[21],"failing":[22],"to":[23,60,76,93],"penalize":[24],"harmful":[25],"contributions":[26],"without":[27],"discouraging":[28],"participation.":[29],"To":[30],"address":[31],"this,":[32],"we":[33],"propose":[34],"the":[35,46,130],"Negativity-Aware":[36],"Incentive":[37],"(NAI)":[38],"mechanism":[39,85],"which":[40],"introduces":[41],"two":[42],"novel":[43],"components.":[44],"First,":[45],"Ability-Result":[47],"Characteristic":[48],"Function":[49],"(AR-CF)":[50],"adapts":[51],"and":[52,65,96,120,161,193],"extends":[53],"Shapley":[54],"value":[55],"theory":[56],"through":[57],"signed":[58],"valuation":[59],"explicitly":[61],"capture":[62],"both":[63],"positive":[64],"negative":[66,185],"contributions,":[67],"by":[68,118,124],"combining":[69],"workers'":[70],"abilities":[71],"with":[72,89,139],"real-time":[73],"task":[74],"results":[75],"define":[77],"contribution":[78],"values.":[79],"Second,":[80],"a":[81,152,162,175],"dynamic":[82],"stake":[83],"pool":[84],"employs":[86],"pre-commitment":[87],"economics":[88],"adaptive":[90],"dual-control":[91],"parameters":[92],"balance":[94],"fairness":[95],"operational":[97],"efficiency.":[98],"Through":[99],"extensive":[100],"experiments":[101],"on":[102,129],"multimodal":[103],"datasets":[104],"(images,":[105],"text,":[106],"audio,":[107],"video),":[108],"NAI":[109],"outperforms":[110],"state-of-the-art":[111],"baselines:":[112],"it":[113],"improves":[114],"video":[115],"accuracy":[117],"16.6%,":[119],"reduces":[121],"fraudulent":[122],"behaviors":[123],"33.9%.":[125],"Furthermore,":[126],"our":[127],"deployment":[128],"NetEase":[131],"Youling":[132],"crowdsourcing":[133,179],"platform,":[134],"serving":[135],"430,000":[136],"registered":[137],"workers":[138],"80,000":[140],"monthly":[141],"active":[142],"workers,":[143],"validates":[144],"NAI's":[145],"real-world":[146],"viability.":[147],"Real-time":[148],"A/B":[149],"testing":[150],"shows":[151],"59.6%":[153],"improvement":[154],"in":[155,165,178],"quality":[157],"beginner":[159],"tasks":[160],"consistent":[163],"reduction":[164],"rates":[167],"(14.8%-33.9%)":[168],"across":[169],"difficulty":[170],"levels.":[171],"This":[172],"work":[173],"establishes":[174],"paradigm":[176],"shift":[177],"system":[180],"design,":[181],"demonstrating":[182],"that":[183],"explicit":[184],"modeling":[186],"can":[187],"enhance":[188],"quality,":[190],"optimize":[191],"costs,":[192],"foster":[194],"participation":[195],"at":[196],"scale.":[197]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-04-28T00:00:00"}