{"id":"https://openalex.org/W7220747381","doi":"https://doi.org/10.1145/3861003","title":"Harmful Content Generation in Text-to-Image Models: Capabilities and Moderation Limitations","display_name":"Harmful Content Generation in Text-to-Image Models: Capabilities and Moderation Limitations","publication_year":2026,"publication_date":"2026-10-29","ids":{"openalex":"https://openalex.org/W7220747381","doi":"https://doi.org/10.1145/3861003"},"language":"en","primary_location":{"id":"doi:10.1145/3861003","doi":"https://doi.org/10.1145/3861003","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3861003","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-c","cwts-core","dongbi-b","jufo-1","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.06503","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116225207","display_name":"Paschalis Giakoumoglou","orcid":"https://orcid.org/0009-0008-4950-023X","observed_orcids":["https://orcid.org/0009-0008-4950-023X"]},"institutions":[{"id":"https://openalex.org/I4210134249","display_name":"Centre for Research and Technology Hellas","ror":"https://ror.org/03bndpq63","country_code":"GR","type":"facility","lineage":["https://openalex.org/I4210134249"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Paschalis Giakoumoglou","raw_affiliation_strings":["Centre for Research and Technology Hellas (CERTH), Greece"],"raw_orcid":"https://orcid.org/0009-0008-4950-023X","affiliations":[{"raw_affiliation_string":"Centre for Research and Technology Hellas (CERTH), Greece","institution_ids":["https://openalex.org/I4210134249"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018362055","display_name":"Manos Schinas","orcid":"https://orcid.org/0009-0007-2132-3430","observed_orcids":["https://orcid.org/0009-0007-2132-3430"]},"institutions":[{"id":"https://openalex.org/I4210134249","display_name":"Centre for Research and Technology Hellas","ror":"https://ror.org/03bndpq63","country_code":"GR","type":"facility","lineage":["https://openalex.org/I4210134249"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Manos Schinas","raw_affiliation_strings":["Centre for Research and Technology Hellas (CERTH), Greece"],"raw_orcid":"https://orcid.org/0009-0007-2132-3430","affiliations":[{"raw_affiliation_string":"Centre for Research and Technology Hellas (CERTH), Greece","institution_ids":["https://openalex.org/I4210134249"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013616365","display_name":"Symeon Papadopoulos","orcid":"https://orcid.org/0000-0002-5441-7341","observed_orcids":["https://orcid.org/0000-0002-5441-7341"]},"institutions":[{"id":"https://openalex.org/I4210134249","display_name":"Centre for Research and Technology Hellas","ror":"https://ror.org/03bndpq63","country_code":"GR","type":"facility","lineage":["https://openalex.org/I4210134249"]}],"countries":["GR"],"is_corresponding":false,"raw_author_name":"Symeon Papadopoulos","raw_affiliation_strings":["Centre for Research and Technology Hellas (CERTH), Greece"],"raw_orcid":"https://orcid.org/0000-0002-5441-7341","affiliations":[{"raw_affiliation_string":"Centre for Research and Technology Hellas (CERTH), Greece","institution_ids":["https://openalex.org/I4210134249"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210134249"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.73236688,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5810999870300293,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.5810999870300293,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.21080000698566437,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.1136000007390976,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/text-to-image-models","display_name":"text-to-image models","score":0.8270000219345093},{"id":"https://openalex.org/keywords/harmful-content-generation","display_name":"harmful content generation","score":0.9319999814033508},{"id":"https://openalex.org/keywords/ai-safety","display_name":"AI safety","score":0.4690000116825104},{"id":"https://openalex.org/keywords/synthetic-image-detection","display_name":"synthetic image detection","score":0.8629999756813049},{"id":"https://openalex.org/keywords/disinformation","display_name":"disinformation","score":0.6290000081062317},{"id":"https://openalex.org/keywords/sexual-content","display_name":"sexual content","score":0.3970000147819519},{"id":"https://openalex.org/keywords/hate-speech","display_name":"hate speech","score":0.9200000166893005},{"id":"https://openalex.org/keywords/training-data-diversity","display_name":"training data diversity","score":0.6430000066757202},{"id":"https://openalex.org/keywords/distributional-bias","display_name":"distributional bias","score":0.46799999475479126}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6484000086784363},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5843999981880188},{"id":"https://openalex.org/C2776552730","wikidata":"https://www.wikidata.org/wiki/Q189656","display_name":"Disinformation","level":3,"score":0.5454000234603882},{"id":"https://openalex.org/C93225998","wikidata":"https://www.wikidata.org/wiki/Q1941972","display_name":"Moderation","level":2,"score":0.5286999940872192},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.453900009393692},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3950999975204468},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.3937000036239624},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.3833000063896179},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.34709998965263367},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3456000089645386},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33799999952316284},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.33250001072883606},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.3221000134944916},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.2773999869823456},{"id":"https://openalex.org/C170477896","wikidata":"https://www.wikidata.org/wiki/Q17039022","display_name":"Ideation","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.26579999923706055},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2653000056743622},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.26089999079704285},{"id":"https://openalex.org/C59519942","wikidata":"https://www.wikidata.org/wiki/Q650665","display_name":"Drone","level":2,"score":0.2590999901294708},{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3861003","doi":"https://doi.org/10.1145/3861003","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3861003","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","ccf-c","cwts-core","dongbi-b","jufo-1","norway-1","poland-140","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2610.06503","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.06503","pdf_url":"https://arxiv.org/pdf/2610.06503","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.06503","doi":"https://doi.org/10.48550/arxiv.2610.06503","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.06503","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.06503","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.06503","pdf_url":"https://arxiv.org/pdf/2610.06503","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.4000000059604645,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G47872817","display_name":null,"funder_award_id":"101214398","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220747381.pdf","grobid_xml":"https://content.openalex.org/works/W7220747381.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7203662861","https://openalex.org/W4388858734","https://openalex.org/W7134192518","https://openalex.org/W4413644452","https://openalex.org/W4394777948","https://openalex.org/W4415796299","https://openalex.org/W4406191680","https://openalex.org/W7139006461","https://openalex.org/W4401867319","https://openalex.org/W7084089785"],"abstract_inverted_index":{"Text-to-image":[0],"generative":[1],"models":[2,47,80,102,147,160,172,213,250],"have":[3],"achieved":[4],"remarkable":[5],"capabilities":[6,150,176],"in":[7,129,222,273,336],"producing":[8,182,217],"photorealistic":[9],"images,":[10],"but":[11],"raise":[12],"concerns":[13],"about":[14],"misuse":[15,296],"for":[16,110,114,120,151,177,191,290,328],"generating":[17,152],"harmful":[18,41,66,104,121,178,220,236],"content.":[19],"While":[20],"safety":[21,83,282,334],"mechanisms":[22,84],"exist,":[23],"systematic":[24],"evaluations":[25],"of":[26,40,96,131,185,224,227,243,297,309],"their":[27],"effectiveness":[28],"against":[29],"realistic":[30,219],"attacks":[31],"remain":[32],"limited.":[33],"This":[34,305],"paper":[35,306],"presents":[36],"a":[37,137],"comprehensive":[38],"assessment":[39,169],"content":[42,91,105,128,237,325],"generation":[43,175,299,339],"across":[44],"modern":[45],"text-to-image":[46,76,298],"using":[48],"an":[49],"automated":[50,228],"pipeline":[51],"that":[52,89,101,171,199,249,316],"transforms":[53],"legitimate":[54],"news":[55],"captions":[56],"into":[57],"unsafe":[58,127,200],"prompts":[59,125,135],"targeting":[60],"sexually":[61,115,165],"explicit":[62,116,166,259],"content,":[63,117,156,181,260,311],"violent":[64,153],"imagery,":[65],"stereotypes,":[67],"self-harm,":[68],"and":[69,85,118,154,193,208,284,313],"hate":[70],"speech.":[71],"We":[72],"evaluate":[73,333],"five":[74],"open":[75],"models,":[77],"including":[78],"standard":[79],"with":[81,210],"built-in":[82],"community":[86,158],"fine-tuned":[87,159],"variants":[88],"bypass":[90],"restrictions.":[92],"Our":[93,241],"human":[94],"evaluation":[95,196,242],"1,500":[97],"generated":[98],"images":[99,221],"reveals":[100,231,248],"produce":[103],"at":[106,301],"concerning":[107],"rates:":[108],"89.2%":[109],"gore-related":[111],"prompts,":[112],"47.6%":[113],"43.6%":[119],"stereotypes.":[122],"Hate":[123],"speech":[124],"elicit":[126],"46.0%":[130],"cases,":[132],"while":[133,157,261],"self-harm":[134],"yield":[136],"lower":[138],"rate":[139],"(34.5%),":[140],"predominantly":[141],"through":[142],"graphic":[143],"violence.":[144],"All":[145,324],"evaluated":[146],"demonstrated":[148],"substantial":[149,235],"stereotypical":[155],"showed":[161],"particular":[162],"vulnerability":[163],"to":[164,188,238,294,332],"prompts.":[167],"Quality":[168],"shows":[170],"largely":[173],"maintain":[174],"versus":[179],"benign":[180,254],"synthetic":[183,244],"imagery":[184],"sufficient":[186],"fidelity":[187],"pose":[189],"risks":[190,335],"disinformation":[192],"abuse.":[194],"Human":[195],"further":[197],"confirms":[198],"prompting":[201],"does":[202],"not":[203],"severely":[204],"impact":[205],"image":[206,245,338],"plausibility":[207],"photorealism,":[209],"more":[211,291],"capable":[212],"such":[214],"as":[215],"FLUX.1-dev":[216],"clearly":[218],"30.9%":[223],"cases.":[225],"Analysis":[226],"moderation":[229],"systems":[230,300],"critical":[232],"limitations,":[233],"allowing":[234],"evade":[239],"detection.":[240],"detection":[246,285],"algorithms":[247],"trained":[251],"solely":[252],"on":[253,258],"datasets":[255],"perform":[256],"worse":[257],"those":[262],"incorporating":[263],"diverse":[264],"training":[265],"data":[266],"improve":[267],"detection,":[268],"highlighting":[269],"semantic":[270],"distribution":[271],"gaps":[272,279],"current":[274,281],"approaches.":[275],"These":[276],"findings":[277],"highlight":[278],"between":[280],"measures":[283],"capabilities,":[286],"demonstrating":[287],"the":[288],"need":[289],"robust":[292],"approaches":[293],"prevent":[295],"scale.":[302],"Content":[303],"Warning:":[304],"contains":[307],"instances":[308],"sexual":[310],"violence,":[312],"hateful":[314],"material":[315],"some":[317],"readers":[318],"may":[319],"find":[320],"offensive":[321],"or":[322],"disturbing.":[323],"is":[326],"presented":[327],"academic":[329],"research":[330],"purposes":[331],"AI":[337],"systems.":[340]},"counts_by_year":[],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2026-10-07T00:00:00"}