{"id":"https://openalex.org/W4308391526","doi":"https://doi.org/10.1145/3548606.3560599","title":"Why So Toxic?","display_name":"Why So Toxic?","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4308391526","doi":"https://doi.org/10.1145/3548606.3560599"},"language":"en","primary_location":{"id":"doi:10.1145/3548606.3560599","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3548606.3560599","pdf_url":null,"source":{"id":"https://openalex.org/S4363608815","display_name":"Proceedings of the 2022 ACM SIGSAC Conference on Computer and Communications Security","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2022 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5109344980","display_name":"Wai Kwong Man","orcid":null},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Wai Man Si","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security, Saarbruceken, Germany"],"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security, Saarbruceken, Germany","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004014878","display_name":"Michael Backes","orcid":"https://orcid.org/0000-0002-9326-6400"},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Michael Backes","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security, Saarbruceken, Germany"],"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security, Saarbruceken, Germany","institution_ids":["https://openalex.org/I4210128801"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052381671","display_name":"Jeremy Blackburn","orcid":"https://orcid.org/0000-0002-4223-1304"},"institutions":[{"id":"https://openalex.org/I123946342","display_name":"Binghamton University","ror":"https://ror.org/008rmbt77","country_code":"US","type":"education","lineage":["https://openalex.org/I123946342"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeremy Blackburn","raw_affiliation_strings":["Binghamton University, Binghamton, NY, USA"],"affiliations":[{"raw_affiliation_string":"Binghamton University, Binghamton, NY, USA","institution_ids":["https://openalex.org/I123946342"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035799220","display_name":"Emiliano De Cristofaro","orcid":"https://orcid.org/0000-0002-7097-6346"},"institutions":[{"id":"https://openalex.org/I45129253","display_name":"University College London","ror":"https://ror.org/02jx3x895","country_code":"GB","type":"education","lineage":["https://openalex.org/I124357947","https://openalex.org/I45129253"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Emiliano De Cristofaro","raw_affiliation_strings":["University College London, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"University College London, London, United Kingdom","institution_ids":["https://openalex.org/I45129253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046881273","display_name":"Gianluca Stringhini","orcid":"https://orcid.org/0000-0002-6162-578X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gianluca Stringhini","raw_affiliation_strings":["Boston University, Boston, MA, USA"],"affiliations":[{"raw_affiliation_string":"Boston University, Boston, MA, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007187227","display_name":"Savvas Zannettou","orcid":"https://orcid.org/0000-0001-5711-1404"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Savvas Zannettou","raw_affiliation_strings":["TU Delft, Delft, Netherlands"],"affiliations":[{"raw_affiliation_string":"TU Delft, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100354592","display_name":"Yang Zhang","orcid":"https://orcid.org/0000-0001-8135-369X"},"institutions":[{"id":"https://openalex.org/I4210128801","display_name":"Helmholtz Center for Information Security","ror":"https://ror.org/02njgxr09","country_code":"DE","type":"facility","lineage":["https://openalex.org/I1305996414","https://openalex.org/I4210128801"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Yang Zhang","raw_affiliation_strings":["CISPA Helmholtz Center for Information Security, Saarbruceken, Germany"],"affiliations":[{"raw_affiliation_string":"CISPA Helmholtz Center for Information Security, Saarbruceken, Germany","institution_ids":["https://openalex.org/I4210128801"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5109344980"],"corresponding_institution_ids":["https://openalex.org/I4210128801"],"apc_list":null,"apc_paid":null,"fwci":3.548,"has_fulltext":false,"cited_by_count":35,"citation_normalized_percentile":{"value":0.94234234,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"2659","last_page":"2673"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.9955999851226807,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11644","display_name":"Spam and Phishing Detection","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.960369348526001},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7473090291023254},{"id":"https://openalex.org/keywords/offensive","display_name":"Offensive","score":0.6978152990341187},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.5942535400390625},{"id":"https://openalex.org/keywords/the-internet","display_name":"The Internet","score":0.5427051782608032},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5404530167579651},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5384867191314697},{"id":"https://openalex.org/keywords/internet-privacy","display_name":"Internet privacy","score":0.48703962564468384},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4493556022644043},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3983278274536133},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.32704734802246094},{"id":"https://openalex.org/keywords/operations-research","display_name":"Operations research","score":0.10194501280784607},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10032191872596741}],"concepts":[{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.960369348526001},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7473090291023254},{"id":"https://openalex.org/C176856949","wikidata":"https://www.wikidata.org/wiki/Q2001676","display_name":"Offensive","level":2,"score":0.6978152990341187},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.5942535400390625},{"id":"https://openalex.org/C110875604","wikidata":"https://www.wikidata.org/wiki/Q75","display_name":"The Internet","level":2,"score":0.5427051782608032},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5404530167579651},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5384867191314697},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.48703962564468384},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4493556022644043},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3983278274536133},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.32704734802246094},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.10194501280784607},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10032191872596741},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3548606.3560599","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3548606.3560599","pdf_url":null,"source":{"id":"https://openalex.org/S4363608815","display_name":"Proceedings of the 2022 ACM SIGSAC Conference on Computer and Communications Security","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2022 ACM SIGSAC Conference on Computer and Communications Security","raw_type":"proceedings-article"},{"id":"pmh:oai:tudelft.nl:uuid:93d66370-6b64-4477-9124-bda1ab6bb1fd","is_oa":false,"landing_page_url":"http://resolver.tudelft.nl/uuid:93d66370-6b64-4477-9124-bda1ab6bb1fd","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1461212393","display_name":null,"funder_award_id":"EP/V011189/1","funder_id":"https://openalex.org/F4320310077","funder_display_name":"National Research Centre"},{"id":"https://openalex.org/G1791190673","display_name":null,"funder_award_id":"CNS-2114407","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G1966752340","display_name":null,"funder_award_id":"ZT-I-OO1 4","funder_id":"https://openalex.org/F4320325698","funder_display_name":"Helmholtz Association"},{"id":"https://openalex.org/G2040152928","display_name":null,"funder_award_id":"EP/V011189/1","funder_id":"https://openalex.org/F4320314731","funder_display_name":"UK Research and Innovation"},{"id":"https://openalex.org/G3913136005","display_name":"Collaborative Research: SaTC: CORE: Small: Detecting Accounts Involved in Influence Campaigns on Social Media","funder_award_id":"2114411","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3971968457","display_name":null,"funder_award_id":"2046590","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4772592055","display_name":null,"funder_award_id":"CNS-1942610","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5921281487","display_name":null,"funder_award_id":"number","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6612803478","display_name":null,"funder_award_id":"IIS-2046590 CNS-2114411 CNS-1942610 CNS-2114407","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6650699768","display_name":null,"funder_award_id":"1942610","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G738021503","display_name":null,"funder_award_id":"CNS-2114411","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320310077","display_name":"National Research Centre","ror":"https://ror.org/02n85j827"},{"id":"https://openalex.org/F4320314731","display_name":"UK Research and Innovation","ror":"https://ror.org/001aqnf71"},{"id":"https://openalex.org/F4320325698","display_name":"Helmholtz Association","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":66,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2042492924","https://openalex.org/W2180612164","https://openalex.org/W2461943168","https://openalex.org/W2535690855","https://openalex.org/W2557436004","https://openalex.org/W2595653137","https://openalex.org/W2604580630","https://openalex.org/W2615146352","https://openalex.org/W2771249222","https://openalex.org/W2785896739","https://openalex.org/W2788277448","https://openalex.org/W2806983856","https://openalex.org/W2890365488","https://openalex.org/W2896457183","https://openalex.org/W2907283777","https://openalex.org/W2952799392","https://openalex.org/W2963206148","https://openalex.org/W2963235411","https://openalex.org/W2964082701","https://openalex.org/W2964910501","https://openalex.org/W2970252402","https://openalex.org/W2971173235","https://openalex.org/W2971307358","https://openalex.org/W2972766508","https://openalex.org/W2982756474","https://openalex.org/W2985799198","https://openalex.org/W2988937804","https://openalex.org/W3008374555","https://openalex.org/W3014514837","https://openalex.org/W3022034311","https://openalex.org/W3024950738","https://openalex.org/W3029778218","https://openalex.org/W3033544963","https://openalex.org/W3033639609","https://openalex.org/W3037924069","https://openalex.org/W3088599783","https://openalex.org/W3101004475","https://openalex.org/W3102516861","https://openalex.org/W3103639864","https://openalex.org/W3123567650","https://openalex.org/W3155584966","https://openalex.org/W3155783507","https://openalex.org/W3171850892","https://openalex.org/W3173465197","https://openalex.org/W3181414820","https://openalex.org/W3183115158","https://openalex.org/W3193741227","https://openalex.org/W3206381865","https://openalex.org/W3207604419","https://openalex.org/W4287887196","https://openalex.org/W4290742115","https://openalex.org/W4312990315","https://openalex.org/W6636805335","https://openalex.org/W6679436768","https://openalex.org/W6680532216","https://openalex.org/W6712785310","https://openalex.org/W6714618025","https://openalex.org/W6739901393","https://openalex.org/W6753443365","https://openalex.org/W6761551260","https://openalex.org/W6773395774","https://openalex.org/W6784187406","https://openalex.org/W6800997799","https://openalex.org/W6810528332","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W4383501580","https://openalex.org/W4214931137","https://openalex.org/W1568520348","https://openalex.org/W3214407891","https://openalex.org/W4313813117","https://openalex.org/W4382052417","https://openalex.org/W3192088754","https://openalex.org/W3194113117","https://openalex.org/W3196817267","https://openalex.org/W1976600725"],"abstract_inverted_index":{"Chatbots":[0],"are":[1,48,76,174,211],"used":[2,217],"in":[3,14,28,68,122],"many":[4],"applications,":[5],"e.g.,":[6],"automated":[7],"agents,":[8],"smart":[9],"home":[10],"assistants,":[11],"interactive":[12],"characters":[13],"online":[15,196],"games,":[16],"etc.":[17],"Therefore,":[18],"it":[19],"is":[20,39,134],"crucial":[21],"to":[22,36,78,101,114,199],"ensure":[23,200],"they":[24,159],"do":[25,204],"not":[26,40,205],"behave":[27],"undesired":[29],"manners,":[30],"providing":[31,79],"offensive":[32],"or":[33,173],"toxic":[34,80,84,94,124],"responses":[35,81,95],"users.":[37,208],"This":[38,59,184],"a":[41,62,123,179],"trivial":[42],"task":[43],"as":[44,218],"state-of-the-art":[45],"chatbot":[46,138,202,236],"models":[47,139,203],"trained":[49],"on":[50,111],"large,":[51],"public":[52,137],"datasets":[53],"openly":[54],"collected":[55],"from":[56,191],"the":[57,162,166,170,182,186,192,228],"Internet.":[58],"paper":[60],"presents":[61],"first-of-its-kind,":[63],"large-scale":[64],"measurement":[65],"of":[66,168,181],"toxicity":[67],"chatbots.":[69],"We":[70,97,149],"show":[71],"that":[72,118,131,158,201,213,223],"publicly":[73],"available":[74],"chatbots":[75,120],"prone":[77],"when":[82],"fed":[83],"queries.":[85],"Even":[86],"more":[87,189,232],"worryingly,":[88],"some":[89],"non-toxic":[90,116],"queries":[91,117,144],"can":[92,215],"trigger":[93],"too.":[96],"then":[98],"set":[99],"out":[100],"design":[102],"and":[103,140,195,222],"experiment":[104],"with":[105],"an":[106,219],"attack,":[107],"ToxicBuddy,":[108,156],"which":[109],"relies":[110],"fine-tuning":[112],"GPT-2":[113],"generate":[115],"make":[119],"respond":[121],"manner.":[125],"Our":[126],"extensive":[127],"experimental":[128],"evaluation":[129],"demonstrates":[130],"our":[132,224],"attack":[133,163],"effective":[135,176,233],"against":[136,155],"outperforms":[141],"manually-crafted":[142],"malicious":[143],"proposed":[145],"by":[146],"previous":[147],"work.":[148],"also":[150],"evaluate":[151],"three":[152],"defense":[153],"mechanisms":[154],"showing":[157],"either":[160],"reduce":[161],"performance":[164],"at":[165,177],"cost":[167],"affecting":[169],"chatbot's":[171],"utility":[172],"only":[175],"mitigating":[178],"portion":[180],"attack.":[183],"highlights":[185],"need":[187],"for":[188,235],"research":[190],"computer":[193],"security":[194],"safety":[197],"communities":[198],"hurt":[206],"their":[207],"Overall,":[209],"we":[210],"confident":[212],"ToxicBuddy":[214],"be":[216],"auditing":[220],"tool":[221],"work":[225],"will":[226],"pave":[227],"way":[229],"toward":[230],"designing":[231],"defenses":[234],"safety.":[237]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":10},{"year":2023,"cited_by_count":6}],"updated_date":"2026-04-11T08:14:18.477133","created_date":"2022-11-11T00:00:00"}