{"id":"https://openalex.org/W2944944282","doi":"https://doi.org/10.1145/3308560.3317083","title":"Crowdsourcing Subjective Tasks: The Case Study of Understanding Toxicity in Online Discussions","display_name":"Crowdsourcing Subjective Tasks: The Case Study of Understanding Toxicity in Online Discussions","publication_year":2019,"publication_date":"2019-05-13","ids":{"openalex":"https://openalex.org/W2944944282","doi":"https://doi.org/10.1145/3308560.3317083","mag":"2944944282"},"language":"en","primary_location":{"id":"doi:10.1145/3308560.3317083","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3308560.3317083","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016381010","display_name":"Lora Aroyo","orcid":"https://orcid.org/0000-0001-9402-1133"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lora Aroyo","raw_affiliation_strings":["Google"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103159534","display_name":"Lucas Dixon","orcid":"https://orcid.org/0000-0003-1094-1675"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lucas Dixon","raw_affiliation_strings":["Jigsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jigsaw","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084311372","display_name":"Nithum Thain","orcid":"https://orcid.org/0000-0002-7367-0916"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nithum Thain","raw_affiliation_strings":["Jigsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jigsaw","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085764332","display_name":"Olivia Redfield","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olivia Redfield","raw_affiliation_strings":["Jigsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jigsaw","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075185179","display_name":"Rachel Rosen","orcid":"https://orcid.org/0000-0003-3949-3060"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rachel Rosen","raw_affiliation_strings":["Jigsaw"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jigsaw","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.496,"has_fulltext":false,"cited_by_count":63,"citation_normalized_percentile":{"value":0.94633534,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1100","last_page":"1105"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9911999702453613,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9889000058174133,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.7418990731239319},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6419122219085693},{"id":"https://openalex.org/keywords/ambiguity","display_name":"Ambiguity","score":0.600072979927063},{"id":"https://openalex.org/keywords/harassment","display_name":"Harassment","score":0.5768155455589294},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.5286320447921753},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5081539154052734},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4907311201095581},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.48321449756622314},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.3876751661300659},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.372088223695755},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.36789482831954956},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.255462646484375},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2055390179157257},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.18963685631752014}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.7418990731239319},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6419122219085693},{"id":"https://openalex.org/C2780522230","wikidata":"https://www.wikidata.org/wiki/Q1140419","display_name":"Ambiguity","level":2,"score":0.600072979927063},{"id":"https://openalex.org/C2778976716","wikidata":"https://www.wikidata.org/wiki/Q3240539","display_name":"Harassment","level":2,"score":0.5768155455589294},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.5286320447921753},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5081539154052734},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4907311201095581},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.48321449756622314},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.3876751661300659},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.372088223695755},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.36789482831954956},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.255462646484375},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2055390179157257},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18963685631752014},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3308560.3317083","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3308560.3317083","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of The 2019 World Wide Web Conference","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.7300000190734863,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W78136081","https://openalex.org/W80056832","https://openalex.org/W150315917","https://openalex.org/W158727920","https://openalex.org/W306109848","https://openalex.org/W1488582764","https://openalex.org/W1554394657","https://openalex.org/W1595709623","https://openalex.org/W1724486527","https://openalex.org/W1962580118","https://openalex.org/W1973306475","https://openalex.org/W1979839410","https://openalex.org/W2018532708","https://openalex.org/W2022881688","https://openalex.org/W2033442452","https://openalex.org/W2034978010","https://openalex.org/W2035707873","https://openalex.org/W2039017419","https://openalex.org/W2044173330","https://openalex.org/W2100738695","https://openalex.org/W2119769989","https://openalex.org/W2119830539","https://openalex.org/W2120308175","https://openalex.org/W2144578442","https://openalex.org/W2148217372","https://openalex.org/W2164124780","https://openalex.org/W2186151411","https://openalex.org/W2290009368","https://openalex.org/W2322584079","https://openalex.org/W2397166854","https://openalex.org/W2540646130","https://openalex.org/W2558888286","https://openalex.org/W2607311634","https://openalex.org/W2610525153","https://openalex.org/W2739912280","https://openalex.org/W2741691725","https://openalex.org/W2756384930","https://openalex.org/W2783668259","https://openalex.org/W2798471701","https://openalex.org/W2888501547","https://openalex.org/W2889482652","https://openalex.org/W2898683381","https://openalex.org/W2904722520","https://openalex.org/W2945608433","https://openalex.org/W2963748066","https://openalex.org/W2963955897","https://openalex.org/W2964235962","https://openalex.org/W3027247846","https://openalex.org/W3103871524","https://openalex.org/W4205184193","https://openalex.org/W4230282492","https://openalex.org/W4285719527","https://openalex.org/W6678162470"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W1497983610","https://openalex.org/W1980158193"],"abstract_inverted_index":{"Discussing":[0],"things":[1],"you":[2,17],"care":[3],"about":[4],"can":[5,175],"be":[6,110,176],"difficult,":[7],"especially":[8],"via":[9],"online":[10,71,81,87,131],"platforms,":[11],"where":[12],"sharing":[13],"your":[14],"opinion":[15],"leaves":[16],"open":[18,209],"to":[19,30,66,199],"the":[20,50,104,107,115,118,126,135,152,215],"real":[21],"and":[22,27,37,52,140,203,211],"immediate":[23],"threats":[24],"of":[25,58,79,106,117,128,137,151,154,182,208],"abuse":[26],"harassment.":[28],"Due":[29],"these":[31,92],"threats,":[32],"people":[33],"stop":[34],"expressing":[35],"themselves":[36],"give":[38],"up":[39],"on":[40,48],"seeking":[41],"different":[42],"opinions.":[43],"Recent":[44],"research":[45,216],"efforts":[46,93],"focus":[47],"examining":[49],"strengths":[51],"weaknesses":[53],"(e.g.":[54,158],"potential":[55],"unintended":[56],"biases)":[57],"using":[59],"machine":[60],"learning":[61],"as":[62,84,170,173],"a":[63,111,206],"support":[64],"tool":[65],"facilitate":[67],"safe":[68],"space":[69],"for":[70,73,114],"discussions;":[72],"example,":[74],"through":[75],"detecting":[76],"various":[77],"types":[78],"negative":[80,119],"behaviors":[82],"such":[83],"hate":[85],"speech,":[86],"harassment,":[88],"or":[89,98,172,184],"cyberbullying.":[90],"Typically,":[91],"build":[94],"upon":[95],"sentiment":[96,163],"analysis":[97,150],"spam":[99],"detection":[100],"in":[101,130,143,218],"text.":[102],"However,":[103],"toxicity":[105,129,161],"language":[108],"could":[109],"strong":[112],"indicator":[113],"intensity":[116],"behavior.":[120],"In":[121],"this":[122,144,201,219],"paper,":[123],"we":[124,167,204],"study":[125],"topic":[127],"conversations":[132],"by":[133,178],"addressing":[134],"problems":[136],"subjectivity,":[138],"bias,":[139],"ambiguity":[141],"inherent":[142],"task.":[145],"We":[146,193],"start":[147],"with":[148],"an":[149],"characteristics":[153],"subjective":[155],"assessment":[156],"tasks":[157],"relevance":[159],"judgment,":[160,162],"assessment,":[164],"etc).":[165],"Whether":[166],"perceive":[168],"something":[169],"relevant":[171],"toxic":[174],"influenced":[177],"almost":[179],"infinite":[180],"amounts":[181],"prior":[183],"current":[185],"context,":[186],"e.g.":[187],"culture,":[188],"background,":[189],"experiences,":[190],"education,":[191],"etc.":[192],"survey":[194],"recent":[195],"work":[196],"that":[197],"tries":[198],"understand":[200],"phenomenon,":[202],"outline":[205],"number":[207],"questions":[210],"challenges":[212],"which":[213],"shape":[214],"perspectives":[217],"multi-disciplinary":[220],"field.":[221]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":16},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}