{"id":"https://openalex.org/W4292590453","doi":"https://doi.org/10.1017/s1351324922000353","title":"Quantifying the impact of context on the quality of manual hate speech annotation","display_name":"Quantifying the impact of context on the quality of manual hate speech annotation","publication_year":2022,"publication_date":"2022-08-22","ids":{"openalex":"https://openalex.org/W4292590453","doi":"https://doi.org/10.1017/s1351324922000353"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324922000353","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324922000353","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1017/s1351324922000353","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5021557805","display_name":"Nikola Ljube\u0161i\u0107","orcid":"https://orcid.org/0000-0001-7169-9152"},"institutions":[{"id":"https://openalex.org/I153976015","display_name":"University of Ljubljana","ror":"https://ror.org/05njb9z20","country_code":"SI","type":"education","lineage":["https://openalex.org/I153976015"]},{"id":"https://openalex.org/I3006985408","display_name":"Jo\u017eef Stefan Institute","ror":"https://ror.org/05060sz93","country_code":"SI","type":"facility","lineage":["https://openalex.org/I3006985408"]}],"countries":["SI"],"is_corresponding":true,"raw_author_name":"Nikola Ljube\u0161i\u0107","raw_affiliation_strings":["Faculty of Computer and Information Science, University of Ljubljana, Ljubljana, Slovenia","Jo\u017eef Stefan Institute, Ljubljana, Slovenia"],"raw_orcid":"https://orcid.org/0000-0001-7169-9152","affiliations":[{"raw_affiliation_string":"Faculty of Computer and Information Science, University of Ljubljana, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I153976015"]},{"raw_affiliation_string":"Jo\u017eef Stefan Institute, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I3006985408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017283574","display_name":"Igor Mozeti\u010d","orcid":"https://orcid.org/0000-0002-5466-0608"},"institutions":[{"id":"https://openalex.org/I3006985408","display_name":"Jo\u017eef Stefan Institute","ror":"https://ror.org/05060sz93","country_code":"SI","type":"facility","lineage":["https://openalex.org/I3006985408"]}],"countries":["SI"],"is_corresponding":false,"raw_author_name":"Igor Mozeti\u010d","raw_affiliation_strings":["Jo\u017eef Stefan Institute, Ljubljana, Slovenia"],"raw_orcid":"https://orcid.org/0000-0002-5466-0608","affiliations":[{"raw_affiliation_string":"Jo\u017eef Stefan Institute, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I3006985408"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5072261415","display_name":"Petra Kralj Novak","orcid":null},"institutions":[{"id":"https://openalex.org/I198945027","display_name":"Central European University","ror":"https://ror.org/02zx40v98","country_code":"AT","type":"education","lineage":["https://openalex.org/I198945027"]},{"id":"https://openalex.org/I3006985408","display_name":"Jo\u017eef Stefan Institute","ror":"https://ror.org/05060sz93","country_code":"SI","type":"facility","lineage":["https://openalex.org/I3006985408"]}],"countries":["AT","SI"],"is_corresponding":false,"raw_author_name":"Petra Kralj Novak","raw_affiliation_strings":["Central European University, Vienna, Austria","Jo\u017eef Stefan Institute, Ljubljana, Slovenia"],"raw_orcid":"https://orcid.org/0000-0003-3385-6430","affiliations":[{"raw_affiliation_string":"Central European University, Vienna, Austria","institution_ids":["https://openalex.org/I198945027"]},{"raw_affiliation_string":"Jo\u017eef Stefan Institute, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I3006985408"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5021557805"],"corresponding_institution_ids":["https://openalex.org/I153976015","https://openalex.org/I3006985408"],"apc_list":null,"apc_paid":null,"fwci":1.1083,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.81493959,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"29","issue":"6","first_page":"1481","last_page":"1494"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10485","display_name":"Bullying, Victimization, and Aggression","score":0.940500020980835,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.9114489555358887},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8665800094604492},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.6801790595054626},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5869325995445251},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5575465559959412},{"id":"https://openalex.org/keywords/schema","display_name":"Schema (genetic algorithms)","score":0.5528758764266968},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5408992171287537},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4447852075099945},{"id":"https://openalex.org/keywords/temporal-annotation","display_name":"Temporal annotation","score":0.4280911982059479},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.346240758895874},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.20738255977630615},{"id":"https://openalex.org/keywords/language-technology","display_name":"Language technology","score":0.09002551436424255}],"concepts":[{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.9114489555358887},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8665800094604492},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.6801790595054626},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5869325995445251},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5575465559959412},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.5528758764266968},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5408992171287537},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4447852075099945},{"id":"https://openalex.org/C7044111","wikidata":"https://www.wikidata.org/wiki/Q15844891","display_name":"Temporal annotation","level":5,"score":0.4280911982059479},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.346240758895874},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.20738255977630615},{"id":"https://openalex.org/C14919245","wikidata":"https://www.wikidata.org/wiki/Q1976109","display_name":"Language technology","level":4,"score":0.09002551436424255},{"id":"https://openalex.org/C129353971","wikidata":"https://www.wikidata.org/wiki/Q5156949","display_name":"Comprehension approach","level":3,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1017/s1351324922000353","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324922000353","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},{"id":"pmh:oai:openresearch.ceu.edu:20.500.14018/27667","is_oa":true,"landing_page_url":"https://hdl.handle.net/20.500.14018/27667","pdf_url":null,"source":{"id":"https://openalex.org/S4306401981","display_name":"Open Research (Auckland University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39854758","host_organization_name":"Auckland University of Technology","host_organization_lineage":["https://openalex.org/I39854758"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Journal article"},{"id":"pmh:oai:repozitorij.uni-lj.si:IzpisGradiva.php?id=155112","is_oa":true,"landing_page_url":"https://repozitorij.uni-lj.si/Dokument.php?id=181892&dn=","pdf_url":"https://repozitorij.uni-lj.si/Dokument.php?id=181893&dn=","source":{"id":"https://openalex.org/S4377196268","display_name":"Repository of the University of Ljubljana (University of Ljubljana)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I153976015","host_organization_name":"University of Ljubljana","host_organization_lineage":["https://openalex.org/I153976015"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Natural language engineering, vol. 29, no. 6, pp. 1481-1494, 2023.","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:https://openresearch.ceu.edu:20.500.14018/28212","is_oa":true,"landing_page_url":"https://hdl.handle.net/20.500.14018/28212","pdf_url":null,"source":{"id":"https://openalex.org/S4306401981","display_name":"Open Research (Auckland University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I39854758","host_organization_name":"Auckland University of Technology","host_organization_lineage":["https://openalex.org/I39854758"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"1494","raw_type":"Journal article"}],"best_oa_location":{"id":"doi:10.1017/s1351324922000353","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324922000353","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7400000095367432,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320273","display_name":"University of Cambridge","ror":"https://ror.org/013meh722"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1857789879","https://openalex.org/W1965443741","https://openalex.org/W2168745915","https://openalex.org/W2740168486","https://openalex.org/W2740496918","https://openalex.org/W2747187574","https://openalex.org/W2887782043","https://openalex.org/W2921834779","https://openalex.org/W2922580172","https://openalex.org/W2954226438","https://openalex.org/W2963532322","https://openalex.org/W2977937308","https://openalex.org/W3013821317","https://openalex.org/W3034951181","https://openalex.org/W3115903740","https://openalex.org/W3146312150","https://openalex.org/W3166927129","https://openalex.org/W3174880282","https://openalex.org/W3175546057","https://openalex.org/W3181126571","https://openalex.org/W3186288536","https://openalex.org/W4210764005","https://openalex.org/W4230965853"],"related_works":["https://openalex.org/W2080575979","https://openalex.org/W4385569300","https://openalex.org/W48671780","https://openalex.org/W1994489417","https://openalex.org/W2369337446","https://openalex.org/W2573587940","https://openalex.org/W2061851981","https://openalex.org/W3145920652","https://openalex.org/W2137228101","https://openalex.org/W166137614"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"quality":[2,89,108],"of":[3,25,45,99,109,192,204,208,211],"annotations":[4,138],"in":[5,36,196,201],"manually":[6,26],"annotated":[7],"hate":[8,15,31,163,181],"speech":[9,16,32,164,182],"datasets":[10],"is":[11,116,127,143,151],"crucial":[12],"for":[13,30,162],"automatic":[14],"detection.":[17],"This":[18,113],"contribution":[19],"focuses":[20],"on":[21,63,106,187],"the":[22,34,38,43,64,75,81,97,107,110,125,188,202,205],"positive":[23,104,114],"effects":[24],"annotating":[27],"online":[28],"comments":[29,39,155],"within":[33],"context":[35,46,100],"which":[37],"occur.":[40],"We":[41,169],"quantify":[42],"impact":[44,105,115],"availability":[47],"by":[48,73],"meticulously":[49],"designing":[50],"an":[51],"experiment:":[52],"Two":[53],"annotation":[54,77,88,177,194],"rounds":[55],"are":[56],"performed,":[57],"one":[58,61],"in-context":[59],"and":[60,79,86,166,183],"out-of-context,":[62],"same":[65,76,82],"English":[66],"YouTube":[67],"data":[68,193],"(more":[69],"than":[70,121],"10,000":[71],"comments),":[72],"using":[74],"schema":[78],"platform,":[80],"highly":[83,159,189],"trained":[84],"annotators,":[85],"quantifying":[87],"through":[90],"inter-annotator":[91],"agreement.":[92],"Our":[93],"results":[94],"show":[95],"that":[96,136,148,171],"presence":[98],"has":[101],"a":[102,158],"significant":[103],"manual":[111],"annotations.":[112],"more":[117],"noticeable":[118],"among":[119,122,154],"replies":[120],"comments,":[123],"although":[124],"former":[126],"harder":[128],"to":[129],"consistently":[130],"annotate":[131],"overall.":[132,168],"Previous":[133],"research":[134,165,186],"reporting":[135],"out-of-context":[137],"favour":[139],"assigning":[140],"non-hate-speech":[141],"labels":[142],"also":[144],"corroborated,":[145],"showing":[146],"further":[147,185],"this":[149,172],"tendency":[150],"especially":[152,200],"present":[153],"inciting":[156],"violence,":[157],"relevant":[160,190],"category":[161],"society":[167],"believe":[170],"work":[173],"will":[174],"improve":[175],"future":[176],"campaigns":[178],"even":[179],"beyond":[180],"motivate":[184],"questions":[191],"methodology":[195],"natural":[197],"language":[198],"processing,":[199],"light":[203],"current":[206],"expansion":[207],"its":[209],"scope":[210],"application.":[212]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}