{"id":"https://openalex.org/W4409750226","doi":"https://doi.org/10.1145/3706598.3713998","title":"Lost in Moderation: How Commercial Content Moderation APIs Over- and Under-Moderate Group-Targeted Hate Speech and Linguistic Variations","display_name":"Lost in Moderation: How Commercial Content Moderation APIs Over- and Under-Moderate Group-Targeted Hate Speech and Linguistic Variations","publication_year":2025,"publication_date":"2025-04-24","ids":{"openalex":"https://openalex.org/W4409750226","doi":"https://doi.org/10.1145/3706598.3713998"},"language":"en","primary_location":{"id":"doi:10.1145/3706598.3713998","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713998","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3706598.3713998","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053533521","display_name":"David Hartmann","orcid":"https://orcid.org/0000-0001-9745-5287"},"institutions":[{"id":"https://openalex.org/I4210117750","display_name":"Weizenbaum Institute","ror":"https://ror.org/023kksk09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I176453806","https://openalex.org/I2800804238","https://openalex.org/I2801702882","https://openalex.org/I315704651","https://openalex.org/I4210117750","https://openalex.org/I4577782","https://openalex.org/I46043019","https://openalex.org/I4923324","https://openalex.org/I75951250"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"David Hartmann","raw_affiliation_strings":["Weizenbaum Institute Berlin, Berlin, Germany and Technische Universit\u00e4t Berlin, Berlin, Germany"],"raw_orcid":"https://orcid.org/0000-0001-9745-5287","affiliations":[{"raw_affiliation_string":"Weizenbaum Institute Berlin, Berlin, Germany and Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4210117750","https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5099380761","display_name":"Amin Oueslati","orcid":null},"institutions":[{"id":"https://openalex.org/I24830596","display_name":"Hertie School","ror":"https://ror.org/0473a4773","country_code":"DE","type":"education","lineage":["https://openalex.org/I24830596"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Amin Oueslati","raw_affiliation_strings":["Hertie School Berlin, Berlin, Germany"],"raw_orcid":"https://orcid.org/0009-0001-6356-2372","affiliations":[{"raw_affiliation_string":"Hertie School Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I24830596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049421291","display_name":"Dimitri Staufer","orcid":"https://orcid.org/0009-0000-1671-222X"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Dimitri Staufer","raw_affiliation_strings":["Technische Universit\u00e4t Berlin, Berlin, Germany"],"raw_orcid":"https://orcid.org/0009-0000-1671-222X","affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103305882","display_name":"Lena Pohlmann","orcid":null},"institutions":[{"id":"https://openalex.org/I4210117750","display_name":"Weizenbaum Institute","ror":"https://ror.org/023kksk09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I176453806","https://openalex.org/I2800804238","https://openalex.org/I2801702882","https://openalex.org/I315704651","https://openalex.org/I4210117750","https://openalex.org/I4577782","https://openalex.org/I46043019","https://openalex.org/I4923324","https://openalex.org/I75951250"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Lena Pohlmann","raw_affiliation_strings":["Weizenbaum Institute Berlin, Berlin, Germany and Technische Universit\u00e4t Berlin, Berlin, Germany"],"raw_orcid":"https://orcid.org/0009-0002-9922-4196","affiliations":[{"raw_affiliation_string":"Weizenbaum Institute Berlin, Berlin, Germany and Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4210117750","https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000465232","display_name":"Simon Munzert","orcid":"https://orcid.org/0000-0003-4760-367X"},"institutions":[{"id":"https://openalex.org/I24830596","display_name":"Hertie School","ror":"https://ror.org/0473a4773","country_code":"DE","type":"education","lineage":["https://openalex.org/I24830596"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Simon Munzert","raw_affiliation_strings":["Hertie School Berlin, Berlin, Germany"],"raw_orcid":"https://orcid.org/0000-0003-4760-367X","affiliations":[{"raw_affiliation_string":"Hertie School Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I24830596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069890392","display_name":"Hendrik Heuer","orcid":"https://orcid.org/0000-0003-1919-9016"},"institutions":[{"id":"https://openalex.org/I167360494","display_name":"University of Wuppertal","ror":"https://ror.org/00613ak93","country_code":"DE","type":"education","lineage":["https://openalex.org/I167360494"]},{"id":"https://openalex.org/I4210145955","display_name":"Center for Advanced Internet Studies","ror":"https://ror.org/04445fp84","country_code":"DE","type":"nonprofit","lineage":["https://openalex.org/I4210145955"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hendrik Heuer","raw_affiliation_strings":["Center for Advanced Internet Studies (CAIS), Bochum, Germany and University of Wuppertal, Wuppertal, Germany"],"raw_orcid":"https://orcid.org/0000-0003-1919-9016","affiliations":[{"raw_affiliation_string":"Center for Advanced Internet Studies (CAIS), Bochum, Germany and University of Wuppertal, Wuppertal, Germany","institution_ids":["https://openalex.org/I4210145955","https://openalex.org/I167360494"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":20.2635,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.99256289,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"26"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/moderation","display_name":"Moderation","score":0.9721863865852356},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.5712777972221375},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.5065799951553345},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4869130849838257},{"id":"https://openalex.org/keywords/content","display_name":"Content (measure theory)","score":0.4813593029975891},{"id":"https://openalex.org/keywords/group","display_name":"Group (periodic table)","score":0.4563610255718231},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.42559126019477844},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.32725855708122253},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.27009260654449463},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.20210155844688416},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.10004124045372009},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.08712881803512573}],"concepts":[{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.9721863865852356},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.5712777972221375},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5065799951553345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4869130849838257},{"id":"https://openalex.org/C2778152352","wikidata":"https://www.wikidata.org/wiki/Q5165061","display_name":"Content (measure theory)","level":2,"score":0.4813593029975891},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.4563610255718231},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.42559126019477844},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32725855708122253},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.27009260654449463},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.20210155844688416},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.10004124045372009},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.08712881803512573},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3706598.3713998","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713998","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2503.01623","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2503.01623","pdf_url":"https://arxiv.org/pdf/2503.01623","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:weizenbaum-library.de:id/901","is_oa":true,"landing_page_url":"https://www.weizenbaum-library.de/handle/id/901","pdf_url":"https://www.weizenbaum-library.de/bitstreams/b8d2b14d-86d9-4f61-8bc2-45c7946fb530/download","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"ConferencePaper"},{"id":"pmh:oai:kobv.de-opus4-hsog:6074","is_oa":false,"landing_page_url":"https://opus4.kobv.de/opus4-hsog/frontdoor/index/index/docId/6074","pdf_url":null,"source":{"id":"https://openalex.org/S4306402087","display_name":"OPUS 4 (Zuse Institute Berlin)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I195893171","host_organization_name":"Zuse Institute Berlin","host_organization_lineage":["https://openalex.org/I195893171"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conferenceobject"}],"best_oa_location":{"id":"doi:10.1145/3706598.3713998","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3706598.3713998","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":84,"referenced_works":["https://openalex.org/W1546751316","https://openalex.org/W2053154970","https://openalex.org/W2473555522","https://openalex.org/W2477388224","https://openalex.org/W2606754711","https://openalex.org/W2730390070","https://openalex.org/W2750779823","https://openalex.org/W2791170418","https://openalex.org/W2897042519","https://openalex.org/W2898761039","https://openalex.org/W2910705748","https://openalex.org/W2920807444","https://openalex.org/W2946681640","https://openalex.org/W2949678053","https://openalex.org/W2956603544","https://openalex.org/W2962059918","https://openalex.org/W2962990575","https://openalex.org/W2963943967","https://openalex.org/W2964235839","https://openalex.org/W2966244612","https://openalex.org/W2970395295","https://openalex.org/W2971307358","https://openalex.org/W2972735048","https://openalex.org/W2980350050","https://openalex.org/W2984063668","https://openalex.org/W2989344603","https://openalex.org/W3008694996","https://openalex.org/W3034937117","https://openalex.org/W3037831233","https://openalex.org/W3045555759","https://openalex.org/W3049565363","https://openalex.org/W3097185012","https://openalex.org/W3099917429","https://openalex.org/W3100279624","https://openalex.org/W3155742828","https://openalex.org/W3159435787","https://openalex.org/W3168175209","https://openalex.org/W3169973903","https://openalex.org/W3173380736","https://openalex.org/W3175487198","https://openalex.org/W3176580738","https://openalex.org/W3201622928","https://openalex.org/W3202183072","https://openalex.org/W3204432444","https://openalex.org/W3206808617","https://openalex.org/W3212327893","https://openalex.org/W4225094305","https://openalex.org/W4285210452","https://openalex.org/W4285606919","https://openalex.org/W4287889469","https://openalex.org/W4289320802","https://openalex.org/W4292296148","https://openalex.org/W4292401525","https://openalex.org/W4317536030","https://openalex.org/W4318621026","https://openalex.org/W4323838292","https://openalex.org/W4327945737","https://openalex.org/W4328008031","https://openalex.org/W4328051875","https://openalex.org/W4365799947","https://openalex.org/W4366547287","https://openalex.org/W4381537659","https://openalex.org/W4385573162","https://openalex.org/W4385734190","https://openalex.org/W4387344829","https://openalex.org/W4389518720","https://openalex.org/W4391099575","https://openalex.org/W4391334942","https://openalex.org/W4391867624","https://openalex.org/W4392646099","https://openalex.org/W4392737162","https://openalex.org/W4394902731","https://openalex.org/W4396815676","https://openalex.org/W4396817230","https://openalex.org/W4396832472","https://openalex.org/W4396832850","https://openalex.org/W4399362631","https://openalex.org/W4399365038","https://openalex.org/W4399528455","https://openalex.org/W4404824525","https://openalex.org/W4411120963","https://openalex.org/W6910851126","https://openalex.org/W6946023902","https://openalex.org/W6962765785"],"related_works":["https://openalex.org/W2740268725","https://openalex.org/W3093710306","https://openalex.org/W750217911","https://openalex.org/W4206949499","https://openalex.org/W2296969234","https://openalex.org/W4226254121","https://openalex.org/W4388229525","https://openalex.org/W131533274","https://openalex.org/W4313163826","https://openalex.org/W1517021018"],"abstract_inverted_index":{"Commercial":[0],"content":[1,63,120],"moderation":[2,64],"APIs":[3,19,77],"are":[4],"marketed":[5],"as":[6,37,85],"scalable":[7],"solutions":[8],"to":[9,29,87,122,162],"combat":[10],"online":[11,31],"hate":[12,89,103],"speech.":[13,90],"However,":[14],"the":[15,54],"reliance":[16],"on":[17,71,80,137,146],"these":[18,164],"risks":[20],"both":[21],"silencing":[22],"legitimate":[23],"speech,":[24,35,104],"called":[25],"over-moderation,":[26],"and":[27,93,119,126,140,143,154,157],"failing":[28],"protect":[30],"platforms":[32],"from":[33],"harmful":[34],"known":[36],"under-moderation.":[38],"To":[39],"assess":[40],"such":[41,84],"risks,":[42],"this":[43],"paper":[44,151],"introduces":[45],"a":[46],"framework":[47],"for":[48,166],"auditing":[49],"black-box":[50],"NLP":[51],"systems.":[52],"Using":[53],"framework,":[55],"we":[56,74],"systematically":[57],"evaluate":[58],"five":[59,67],"widely":[60],"used":[61],"commercial":[62],"APIs.":[65],"Analyzing":[66],"million":[68],"queries":[69],"based":[70],"four":[72],"datasets,":[73],"find":[75],"that":[76,131],"frequently":[78],"rely":[79],"group":[81],"identity":[82],"terms,":[83],"\u201cblack\u201d,":[86],"predict":[88],"While":[91],"OpenAI\u2019s":[92],"Amazon\u2019s":[94],"services":[95],"perform":[96],"slightly":[97],"better,":[98],"all":[99],"providers":[100,133],"under-moderate":[101],"implicit":[102],"which":[105],"uses":[106],"codified":[107],"messages,":[108],"especially":[109],"against":[110],"LGBTQIA+":[111],"individuals.":[112],"Simultaneously,":[113],"they":[114],"over-moderate":[115],"counter-speech,":[116],"reclaimed":[117],"slurs":[118],"related":[121],"Black,":[123],"LGBTQIA+,":[124],"Jewish,":[125],"Muslim":[127],"people.":[128],"We":[129,159],"recommend":[130],"API":[132,138],"offer":[134],"better":[135],"guidance":[136],"implementation":[139],"threshold":[141],"setting":[142],"more":[144],"transparency":[145],"their":[147],"APIs\u2019":[148],"limitations.Warning:":[149],"This":[150],"contains":[152],"offensive":[153],"hateful":[155],"terms":[156,165],"concepts.":[158],"have":[160],"chosen":[161],"reproduce":[163],"reasons":[167],"of":[168],"transparency.":[169]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-10T00:00:00"}