{"id":"https://openalex.org/W4385568418","doi":"https://doi.org/10.1145/3580305.3599828","title":"From Labels to Decisions: A Mapping-Aware Annotator Model","display_name":"From Labels to Decisions: A Mapping-Aware Annotator Model","publication_year":2023,"publication_date":"2023-08-04","ids":{"openalex":"https://openalex.org/W4385568418","doi":"https://doi.org/10.1145/3580305.3599828"},"language":"en","primary_location":{"id":"doi:10.1145/3580305.3599828","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3580305.3599828","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3580305.3599828","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3580305.3599828","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103043927","display_name":"Evan Yao","orcid":"https://orcid.org/0009-0000-6192-3229"},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Evan Yao","raw_affiliation_strings":["Massachusetts Institute of Technology, Cambridge, MA, USA"],"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology, Cambridge, MA, USA","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101564490","display_name":"Jagdish Ramakrishnan","orcid":"https://orcid.org/0009-0009-3299-4613"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jagdish Ramakrishnan","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100385771","display_name":"Xu Chen","orcid":"https://orcid.org/0000-0003-3422-0127"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu Chen","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080426723","display_name":"Viet-An Nguyen","orcid":"https://orcid.org/0000-0002-7923-0882"},"institutions":[{"id":"https://openalex.org/I4210099336","display_name":"Menlo School","ror":"https://ror.org/01240pn49","country_code":"US","type":"education","lineage":["https://openalex.org/I4210099336"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Viet-An Nguyen","raw_affiliation_strings":["Meta, Menlo Park, CA, USA"],"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, CA, USA","institution_ids":["https://openalex.org/I4210099336"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5000980244","display_name":"Udi Weinsberg","orcid":"https://orcid.org/0000-0002-6966-1945"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Udi Weinsberg","raw_affiliation_strings":["Meta, Menlo Park, MA, USA"],"affiliations":[{"raw_affiliation_string":"Meta, Menlo Park, MA, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5103043927"],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.09241752,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5404","last_page":"5415"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9959999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8388439416885376},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7802170515060425},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.6424859166145325},{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.5061041712760925},{"id":"https://openalex.org/keywords/parameterized-complexity","display_name":"Parameterized complexity","score":0.5047348737716675},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.492241770029068},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4738084375858307},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.4467747211456299},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.43692493438720703},{"id":"https://openalex.org/keywords/population","display_name":"Population","score":0.41631361842155457},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12519019842147827}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8388439416885376},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7802170515060425},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6424859166145325},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.5061041712760925},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.5047348737716675},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.492241770029068},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4738084375858307},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4467747211456299},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.43692493438720703},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.41631361842155457},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12519019842147827},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3580305.3599828","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3580305.3599828","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3580305.3599828","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3580305.3599828","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3580305.3599828","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3580305.3599828","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385568418.pdf","grobid_xml":"https://content.openalex.org/works/W4385568418.grobid-xml"},"referenced_works_count":18,"referenced_works":["https://openalex.org/W9014458","https://openalex.org/W1541280084","https://openalex.org/W1972675781","https://openalex.org/W2141649520","https://openalex.org/W2180771322","https://openalex.org/W2330025869","https://openalex.org/W2577537660","https://openalex.org/W2585226541","https://openalex.org/W2597289420","https://openalex.org/W2792932412","https://openalex.org/W2899689163","https://openalex.org/W2914958722","https://openalex.org/W3080755960","https://openalex.org/W3114228236","https://openalex.org/W4206834956","https://openalex.org/W4290944780","https://openalex.org/W4292092773","https://openalex.org/W4292341621"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W2276146857"],"abstract_inverted_index":{"Online":[0],"platforms":[1],"regularly":[2],"rely":[3],"on":[4,224],"human":[5],"annotators":[6,34,62,183],"to":[7,77,91,118,137,146,184,191],"make":[8,119],"real-time":[9],"operational":[10,57],"decisions":[11,58,78,101,138],"for":[12,24,162,215],"tasks":[13],"such":[14,87],"as":[15],"content":[16,85,106,225],"moderation.":[17],"While":[18],"crowdsourcing":[19,52],"models":[20],"have":[21],"been":[22],"proposed":[23,204,230],"aggregating":[25],"noisy":[26],"labels,":[27],"they":[28],"do":[29],"not":[30],"generalize":[31],"well":[32,214],"when":[33],"produce":[35,63],"a":[36,39,50,66,80,157,174,221,233],"labels":[37,64,88,136],"in":[38,65,173,236,245],"large":[40,218],"space,":[41],"e.g.,":[42],"generated":[43],"from":[44,140],"complex":[45],"review":[46],"trees.":[47],"We":[48,155],"study":[49],"novel":[51,158],"setting":[53],"with":[54,177],"D":[55,73],"possible":[56],"or":[59,107],"outcomes,":[60],"but":[61],"larger":[67],"space":[68,99],"of":[69,100,143],"size":[70],"L":[71],">":[72],"which":[74],"are":[75],"mapped":[76],"through":[79],"known":[81],"mapping":[82,139],"function.":[83],"For":[84],"moderation,":[86],"can":[89],"correspond":[90],"violation":[92],"reasons":[93],"(e.g.":[94],"nudity,":[95],"violence),":[96],"while":[97],"the":[98,105,120,126,135,141,203,229],"is":[102,115,171],"binary:":[103],"remove":[104],"keep":[108],"it":[109,114],"up.":[110],"In":[111,220],"this":[112,167],"setting,":[113],"more":[116],"important":[117],"right":[121],"decision":[122],"rather":[123],"than":[124],"estimating":[125],"correct":[127],"underlying":[128],"label.":[129],"Existing":[130],"methods":[131,211],"typically":[132],"separate":[133],"out":[134],"modeling":[142],"annotators,":[144],"leading":[145],"sub-optimal":[147],"statistical":[148],"inference":[149],"efficiency":[150],"and":[151,188,212,217],"excessive":[152],"computation":[153],"complexity.":[154],"propose":[156],"confusion":[159],"matrix":[160],"model":[161,170,185,205,244],"each":[163],"annotator":[164],"that":[165,202],"leverages":[166],"mapping.":[168],"Our":[169],"parameterized":[172],"hierarchical":[175],"manner":[176],"both":[178],"population":[179],"parameters":[180,190],"shared":[181,186],"across":[182],"confusions":[187],"individual":[189],"admit":[192],"heterogeneity":[193],"among":[194],"annotators.":[195],"With":[196],"extensive":[197],"numerical":[198],"experiments,":[199],"we":[200],"demonstrate":[201],"substantially":[206],"improves":[207],"accuracy":[208],"over":[209,238],"existing":[210,243],"scales":[213],"moderate":[216],"L.":[219],"real-world":[222],"application":[223],"moderation":[226],"at":[227],"Meta,":[228],"method":[231],"offers":[232],"13%":[234],"improvement":[235],"AUC":[237],"prior":[239],"methods,":[240],"including":[241],"Meta's":[242],"production.":[246]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}