{"id":"https://openalex.org/W4415536739","doi":"https://doi.org/10.1145/3746027.3754574","title":"Self-Aware Safety Augmentation: Leveraging Internal Semantic Understanding to Enhance Safety in Vision-Language Models","display_name":"Self-Aware Safety Augmentation: Leveraging Internal Semantic Understanding to Enhance Safety in Vision-Language Models","publication_year":2025,"publication_date":"2025-10-25","ids":{"openalex":"https://openalex.org/W4415536739","doi":"https://doi.org/10.1145/3746027.3754574"},"language":null,"primary_location":{"id":"doi:10.1145/3746027.3754574","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3754574","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Wanying Wang","orcid":"https://orcid.org/0000-0002-0452-7593"},"institutions":[{"id":"https://openalex.org/I4405261568","display_name":"Shanghai Key Laboratory of Computer Software Testing and Evaluating","ror":"https://ror.org/05gky6m26","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4405261568","https://openalex.org/I4405262724"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wanying Wang","raw_affiliation_strings":["Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-0452-7593","affiliations":[{"raw_affiliation_string":"Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China","institution_ids":["https://openalex.org/I4405261568"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100575465","display_name":"Zeyu Ma","orcid":"https://orcid.org/0009-0002-5154-0392"},"institutions":[{"id":"https://openalex.org/I21945476","display_name":"Shanghai Normal University","ror":"https://ror.org/01cxqmw89","country_code":"CN","type":"education","lineage":["https://openalex.org/I21945476"]},{"id":"https://openalex.org/I4405261568","display_name":"Shanghai Key Laboratory of Computer Software Testing and Evaluating","ror":"https://ror.org/05gky6m26","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4405261568","https://openalex.org/I4405262724"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyu Ma","raw_affiliation_strings":["Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China and Shanghai Normal University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0002-5154-0392","affiliations":[{"raw_affiliation_string":"Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China and Shanghai Normal University, Shanghai, China","institution_ids":["https://openalex.org/I21945476","https://openalex.org/I4405261568"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Han Zheng","orcid":"https://orcid.org/0009-0006-6668-8145"},"institutions":[{"id":"https://openalex.org/I4210128805","display_name":"MediaTek (Singapore)","ror":"https://ror.org/031y85330","country_code":"SG","type":"company","lineage":["https://openalex.org/I4210128805","https://openalex.org/I4210148979"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Han Zheng","raw_affiliation_strings":["TrustAI Pte. Ltd., Singapore, Singapore"],"raw_orcid":"https://orcid.org/0009-0006-6668-8145","affiliations":[{"raw_affiliation_string":"TrustAI Pte. Ltd., Singapore, Singapore","institution_ids":["https://openalex.org/I4210128805"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080281882","display_name":"Xin Tan","orcid":"https://orcid.org/0000-0001-9346-1196"},"institutions":[{"id":"https://openalex.org/I66867065","display_name":"East China Normal University","ror":"https://ror.org/02n96ep67","country_code":"CN","type":"education","lineage":["https://openalex.org/I66867065"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Tan","raw_affiliation_strings":["East China Normal University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-9346-1196","affiliations":[{"raw_affiliation_string":"East China Normal University, Shanghai, China","institution_ids":["https://openalex.org/I66867065"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029237672","display_name":"Mingang Chen","orcid":"https://orcid.org/0000-0002-0718-8187"},"institutions":[{"id":"https://openalex.org/I4405261568","display_name":"Shanghai Key Laboratory of Computer Software Testing and Evaluating","ror":"https://ror.org/05gky6m26","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4405261568","https://openalex.org/I4405262724"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingang Chen","raw_affiliation_strings":["Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-0718-8187","affiliations":[{"raw_affiliation_string":"Shanghai Key Laboratory of Computer Software Testing and Evaluating, Shanghai, China","institution_ids":["https://openalex.org/I4405261568"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.26897722,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"11199","last_page":"11208"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9782999753952026,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9627000093460083,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.541700005531311},{"id":"https://openalex.org/keywords/framing","display_name":"Framing (construction)","score":0.517799973487854},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.38999998569488525},{"id":"https://openalex.org/keywords/semantic-memory","display_name":"Semantic memory","score":0.3774000108242035},{"id":"https://openalex.org/keywords/semantic-data-model","display_name":"Semantic data model","score":0.36000001430511475},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"Semantic similarity","score":0.35670000314712524},{"id":"https://openalex.org/keywords/semantic-computing","display_name":"Semantic computing","score":0.35199999809265137}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6977999806404114},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.541700005531311},{"id":"https://openalex.org/C169087156","wikidata":"https://www.wikidata.org/wiki/Q2131593","display_name":"Framing (construction)","level":2,"score":0.517799973487854},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.38999998569488525},{"id":"https://openalex.org/C197914299","wikidata":"https://www.wikidata.org/wiki/Q18650","display_name":"Semantic memory","level":3,"score":0.3774000108242035},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36820000410079956},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C511149849","wikidata":"https://www.wikidata.org/wiki/Q7449051","display_name":"Semantic computing","level":3,"score":0.35199999809265137},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3465000092983246},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33820000290870667},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.33739998936653137},{"id":"https://openalex.org/C6881194","wikidata":"https://www.wikidata.org/wiki/Q7449091","display_name":"Semantic technology","level":4,"score":0.3273000121116638},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3203999996185303},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.28839999437332153},{"id":"https://openalex.org/C2778493491","wikidata":"https://www.wikidata.org/wiki/Q7449072","display_name":"Semantic matching","level":3,"score":0.28619998693466187},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.2619999945163727}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3746027.3754574","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3746027.3754574","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 33rd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W4390190013","https://openalex.org/W4390190202","https://openalex.org/W4393157467","https://openalex.org/W4396568157","https://openalex.org/W4402670707","https://openalex.org/W4403345968","https://openalex.org/W4404575054"],"related_works":[],"abstract_inverted_index":{"Large":[0],"vision-language":[1],"models":[2],"(LVLMs)":[3],"are":[4],"vulnerable":[5],"to":[6,10,72,109,120,127],"harmful":[7],"input":[8],"compared":[9],"their":[11,24,52],"language-only":[12],"backbones.":[13],"We":[14],"investigated":[15],"this":[16],"vulnerability":[17],"by":[18,78],"exploring":[19],"LVLMs":[20],"internal":[21,124],"dynamics,":[22],"framing":[23],"inherent":[25,106],"safety":[26,40,63,111,148],"understanding":[27,108],"in":[28,75],"terms":[29],"of":[30,149],"three":[31],"key":[32],"capabilities.":[33],"Specifically,":[34],"we":[35,81,116],"define":[36],"these":[37,79],"capabilities":[38],"as":[39],"perception,":[41],"semantic":[42,69,92,107,125],"understanding,":[43,70],"and":[44,49,140],"alignment":[45],"for":[46],"linguistic":[47],"expression,":[48],"experimentally":[50],"pinpointed":[51],"primary":[53],"locations":[54],"within":[55],"the":[56,73,104,122,129,132,147,155],"model":[57],"architecture.":[58],"The":[59],"results":[60],"indicate":[61],"that":[62,89,143],"perception":[64],"often":[65],"emerges":[66],"before":[67,131],"comprehensive":[68],"leading":[71],"reduction":[74],"safety.":[76],"Motivated":[77],"findings,":[80],"propose":[82],"Self-Aware":[83],"Safety":[84],"Augmentation":[85],"(SASA),":[86],"a":[87],"technique":[88],"projects":[90],"informative":[91],"representations":[93],"from":[94],"intermediate":[95],"layers":[96],"onto":[97],"earlier":[98],"safety-oriented":[99],"layers.":[100],"This":[101],"approach":[102],"leverages":[103],"model's":[105,123],"enhance":[110],"recognition":[112],"without":[113],"fine-tuning.":[114],"Then,":[115],"employ":[117],"linear":[118],"probing":[119],"articulate":[121],"comprehension":[126],"detect":[128],"risk":[130],"generation":[133],"process.":[134],"Extensive":[135],"experiments":[136],"on":[137,154],"various":[138],"datasets":[139],"tasks":[141],"demonstrate":[142],"SASA":[144],"significantly":[145],"improves":[146],"LVLMs,":[150],"with":[151],"minimal":[152],"impact":[153],"utility.":[156]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-25T00:00:00"}