{"id":"https://openalex.org/W3037594992","doi":"https://doi.org/10.48550/arxiv.2006.13507","title":"On Analyzing Annotation Consistency in Online Abusive Behavior Datasets","display_name":"On Analyzing Annotation Consistency in Online Abusive Behavior Datasets","publication_year":2020,"publication_date":"2020-06-24","ids":{"openalex":"https://openalex.org/W3037594992","doi":"https://doi.org/10.48550/arxiv.2006.13507","mag":"3037594992"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2006.13507","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.13507","pdf_url":"https://arxiv.org/pdf/2006.13507","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2006.13507","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023174812","display_name":"Rabiul Awal","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Awal, Md Rabiul","raw_affiliation_strings":["University of Saskatchewan , Canada","University of Saskatchewan >  >  >  >"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Saskatchewan , Canada","institution_ids":["https://openalex.org/I32625721"]},{"raw_affiliation_string":"University of Saskatchewan >  >  >  >","institution_ids":["https://openalex.org/I32625721"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077167782","display_name":"Rui Cao","orcid":"https://orcid.org/0000-0002-9444-6913","observed_orcids":["https://orcid.org/0000-0002-9444-6913","https://orcid.org/0009-0006-6182-1381"]},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cao, Rui","raw_affiliation_strings":["University of Electronic Science and Technology of China , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China , China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089793938","display_name":"Roy Ka-Wei Lee","orcid":"https://orcid.org/0000-0002-1986-7750","observed_orcids":["https://orcid.org/0000-0002-1986-7750"]},"institutions":[{"id":"https://openalex.org/I32625721","display_name":"University of Saskatchewan","ror":"https://ror.org/010x8gc63","country_code":"CA","type":"education","lineage":["https://openalex.org/I32625721"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Lee, Roy Ka-Wei","raw_affiliation_strings":["University of Saskatchewan >  >  >  >"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Saskatchewan >  >  >  >","institution_ids":["https://openalex.org/I32625721"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5021087370","display_name":"Sandra Mitrovi\u0107","orcid":"https://orcid.org/0000-0002-5697-5865","observed_orcids":["https://orcid.org/0000-0002-5697-5865"]},"institutions":[{"id":"https://openalex.org/I2614128279","display_name":"Dalle Molle Institute for Artificial Intelligence Research","ror":"https://ror.org/013355g38","country_code":"CH","type":"facility","lineage":["https://openalex.org/I15196421","https://openalex.org/I2614128279","https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Mitrovi\u0107, Sandra","raw_affiliation_strings":["Dalle Molle Institute for Artificial Intelligence Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Dalle Molle Institute for Artificial Intelligence Research","institution_ids":["https://openalex.org/I2614128279"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":17,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.0,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/annotation-consistency","display_name":"annotation consistency","score":0.9359999895095825},{"id":"https://openalex.org/keywords/online-hate-speech","display_name":"online hate speech","score":0.9390000104904175},{"id":"https://openalex.org/keywords/annotated-data","display_name":"annotated dataset","score":0.8880000114440918},{"id":"https://openalex.org/keywords/label-ambiguity","display_name":"label ambiguity","score":0.5230000019073486},{"id":"https://openalex.org/keywords/dataset-evaluation","display_name":"dataset evaluation","score":0.5360000133514404}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.811030387878418},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.8067876696586609},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6378370523452759},{"id":"https://openalex.org/C518677369","wikidata":"https://www.wikidata.org/wiki/Q202833","display_name":"Social media","level":2,"score":0.5678008794784546},{"id":"https://openalex.org/C14641543","wikidata":"https://www.wikidata.org/wiki/Q553270","display_name":"Group cohesiveness","level":2,"score":0.526466429233551},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4163680970668793},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3885790705680847},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33175307512283325},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27424490451812744},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.1892288625240326},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.15971195697784424},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2006.13507","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.13507","pdf_url":"https://arxiv.org/pdf/2006.13507","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2006.13507","doi":"https://doi.org/10.48550/arxiv.2006.13507","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2006.13507","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2006.13507","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.13507","pdf_url":"https://arxiv.org/pdf/2006.13507","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3037594992.pdf","grobid_xml":"https://content.openalex.org/works/W3037594992.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2250539671","https://openalex.org/W2473555522","https://openalex.org/W2563826943","https://openalex.org/W2595653137","https://openalex.org/W2740168486","https://openalex.org/W2887782043","https://openalex.org/W2903015761","https://openalex.org/W2914707617","https://openalex.org/W6930582848"],"related_works":["https://openalex.org/W6929292895","https://openalex.org/W4361288051","https://openalex.org/W3212536292","https://openalex.org/W7018030796","https://openalex.org/W3030681628","https://openalex.org/W3186955206","https://openalex.org/W4386462154","https://openalex.org/W4385569797","https://openalex.org/W7094832921","https://openalex.org/W2785615365"],"abstract_inverted_index":{"Online":[0],"abusive":[1,36,54,92,115,143],"behavior":[2,144],"is":[3,62,67,150],"an":[4,103],"important":[5],"issue":[6],"that":[7,134,148],"breaks":[8],"the":[9,47,57,74,82,86,108,125,128,159,164],"cohesiveness":[10],"of":[11,59,77,85,127,155],"online":[12,35,50,112,139],"social":[13],"communities":[14],"and":[15,33,53,93,95,114,142],"even":[16],"raises":[17],"public":[18],"safety":[19],"concerns":[20],"in":[21,45,111,130,138,158],"our":[22,120],"societies.":[23],"Motivated":[24],"by":[25],"this":[26,99],"rising":[27],"issue,":[28],"researchers":[29],"have":[30],"proposed,":[31],"collected,":[32],"annotated":[34],"content":[37,116],"datasets.":[38,117],"These":[39],"datasets":[40,61,133],"play":[41],"a":[42,63,78,152],"critical":[43],"role":[44],"facilitating":[46],"research":[48],"on":[49,70],"hate":[51,113,140],"speech":[52,141],"behaviors.":[55],"However,":[56],"annotation":[58,109,129,156],"such":[60],"difficult":[64],"task;":[65],"it":[66],"often":[68,96],"contentious":[69],"what":[71],"should":[72],"be":[73,89],"true":[75],"label":[76],"given":[79],"text":[80],"as":[81],"semantic":[83],"difference":[84],"labels":[87,165],"may":[88],"blurred":[90],"(e.g.,":[91],"hate)":[94],"subjective.":[97],"In":[98],"study,":[100],"we":[101],"proposed":[102,121],"analytical":[104],"framework":[105,122],"to":[106,123],"study":[107],"consistency":[110,126],"We":[118,146],"applied":[119],"evaluate":[124],"three":[131],"popular":[132],"are":[135,166],"widely":[136],"used":[137],"studies.":[145],"found":[147],"there":[149],"still":[151],"substantial":[153],"amount":[154],"inconsistency":[157],"existing":[160],"datasets,":[161],"particularly":[162],"when":[163],"semantically":[167],"similar.":[168]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2020-07-02T00:00:00"}