{"id":"https://openalex.org/W4412197314","doi":"https://doi.org/10.1145/3725899.3725911","title":"A MoE-based Safety Fine-tuning Method for Multimodal Large Language Models","display_name":"A MoE-based Safety Fine-tuning Method for Multimodal Large Language Models","publication_year":2025,"publication_date":"2025-01-10","ids":{"openalex":"https://openalex.org/W4412197314","doi":"https://doi.org/10.1145/3725899.3725911"},"language":"en","primary_location":{"id":"doi:10.1145/3725899.3725911","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725899.3725911","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 8th International Conference on Software Engineering and Information Management","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3725899.3725911","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114168335","display_name":"Runjia Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Runjia Zhang","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0009-0003-7096-1739","affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006750623","display_name":"Qi Wen Xue","orcid":null},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Xue","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0009-0006-0917-3399","affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068430879","display_name":"N. Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087087","display_name":"Henan Tianguan Group (China)","ror":"https://ror.org/001rfde81","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210087087"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nanxin Zhang","raw_affiliation_strings":["Tianfu Jiangxi Laboratory, Chengdu, China"],"raw_orcid":"https://orcid.org/0009-0006-1901-7728","affiliations":[{"raw_affiliation_string":"Tianfu Jiangxi Laboratory, Chengdu, China","institution_ids":["https://openalex.org/I4210087087"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yu Wang","orcid":"https://orcid.org/0009-0007-9891-2348"},"institutions":[{"id":"https://openalex.org/I4210087087","display_name":"Henan Tianguan Group (China)","ror":"https://ror.org/001rfde81","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210087087"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Wang","raw_affiliation_strings":["Tianfu Jiangxi Laboratory, Chengdu, China"],"raw_orcid":"https://orcid.org/0009-0007-9891-2348","affiliations":[{"raw_affiliation_string":"Tianfu Jiangxi Laboratory, Chengdu, China","institution_ids":["https://openalex.org/I4210087087"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029314634","display_name":"Xiurui Xie","orcid":"https://orcid.org/0000-0002-3720-4379"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiurui Xie","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-3720-4379","affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020073761","display_name":"Dongyang Zhang","orcid":"https://orcid.org/0000-0002-4839-0234"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongyang Zhang","raw_affiliation_strings":["University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-4839-0234","affiliations":[{"raw_affiliation_string":"University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5114168335"],"corresponding_institution_ids":["https://openalex.org/I150229711"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.07884783,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"72","last_page":"78"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9926999807357788,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9332000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6958858966827393},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3669268488883972},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.35724446177482605}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6958858966827393},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3669268488883972},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.35724446177482605}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3725899.3725911","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725899.3725911","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 8th International Conference on Software Engineering and Information Management","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3725899.3725911","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3725899.3725911","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2025 8th International Conference on Software Engineering and Information Management","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.41999998688697815}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W2150884987","https://openalex.org/W3119866685","https://openalex.org/W3170943566","https://openalex.org/W4226278401","https://openalex.org/W4226283120","https://openalex.org/W4282028729","https://openalex.org/W4307079201","https://openalex.org/W4378768739","https://openalex.org/W4383993628","https://openalex.org/W4387583347","https://openalex.org/W4387787572","https://openalex.org/W4389520274","https://openalex.org/W4392592030","https://openalex.org/W4402671841","https://openalex.org/W4404356490","https://openalex.org/W4406232026","https://openalex.org/W6761041305"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,12,97,181,186],"Multimodal":[3],"Large":[4],"Language":[5],"Models":[6],"(MLLMs)":[7],"have":[8,191],"demonstrated":[9],"remarkable":[10],"capabilities":[11],"understanding":[13],"and":[14,73,111,184],"reasoning":[15],"across":[16],"different":[17],"modalities.":[18],"Currently,":[19],"the":[20,23,48,98,129,161,168,174,201],"performance":[21,77,154],"of":[22,43,176,203],"model":[24,44,71],"on":[25,52,118,155,160,206],"downstream":[26,54,75,114,156],"tasks":[27],"is":[28],"mainly":[29],"improved":[30],"by":[31,84,164],"various":[32],"fine-tuning":[33,37,49,63,144],"methods,":[34],"however,":[35],"traditional":[36,138],"methods":[38,145],"often":[39,146],"lead":[40],"to":[41,108,113,134,137,167,200],"degradation":[42],"safety,":[45],"even":[46],"when":[47],"process":[50],"focuses":[51],"benign":[53],"tasks.":[55,115],"This":[56],"paper":[57],"introduces":[58],"an":[59,120],"expert":[60,81,178],"mixture-based":[61],"safety":[62,72,109,183],"method":[64,127],"for":[65],"multimodal":[66,122,187],"models":[67,205],"that":[68,125],"simultaneously":[69],"maintains":[70],"enhances":[74],"task":[76,148],"through":[78],"multiple":[79],"specialized":[80],"adapters,":[82],"coordinated":[83],"a":[85],"learnable":[86],"dynamic":[87],"routing":[88],"mechanism.":[89],"Our":[90],"approach":[91,151,180],"incorporates":[92],"eight":[93],"LoRA":[94],"adapter":[95],"blocks":[96],"vision":[99],"encoder's":[100],"attention":[101],"layers,":[102],"where":[103],"four":[104,112],"adapters":[105],"are":[106],"dedicated":[107],"control":[110],"Experimental":[116],"results":[117,172],"MiniCPM-V2.6,":[119],"8B-parameter":[121],"model,":[123],"demonstrate":[124],"our":[126,150,177],"reduces":[128],"harmfulness":[130],"rate":[131],"from":[132],"33.08%":[133],"30.45%":[135],"compared":[136,166],"LoRA.":[139],"Notably,":[140],"while":[141],"conventional":[142],"safety-oriented":[143],"sacrifice":[147],"performance,":[149],"achieves":[152],"superior":[153],"tasks,":[157],"improving":[158],"helpfulness":[159],"TextVQA":[162],"dataset":[163],"19.44%":[165],"base":[169],"model.":[170],"These":[171],"validate":[173],"effectiveness":[175],"mixture":[179],"balancing":[182],"utility":[185],"models.":[188],"Besides,":[189],"we":[190],"also":[192],"conducted":[193],"domestic":[194,208],"hardware":[195,209],"platform":[196],"adaptation":[197],"experiments,":[198],"contributing":[199],"application":[202],"large":[204],"Chinese":[207],"platforms.":[210]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}