{"id":"https://openalex.org/W4403791781","doi":"https://doi.org/10.1145/3664647.3681014","title":"Semantic Alignment for Multimodal Large Language Models","display_name":"Semantic Alignment for Multimodal Large Language Models","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403791781","doi":"https://doi.org/10.1145/3664647.3681014"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681014","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Tao Wu","orcid":"https://orcid.org/0009-0006-7917-8849"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tao Wu","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0009-0006-7917-8849","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101737334","display_name":"Mengze Li","orcid":"https://orcid.org/0000-0002-3482-234X"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mengze Li","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-3482-234X","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009749449","display_name":"Jingyuan Chen","orcid":"https://orcid.org/0000-0003-0415-6937"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingyuan Chen","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-0415-6937","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101397422","display_name":"Wei Ji","orcid":"https://orcid.org/0000-0002-8106-9768"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Wei Ji","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-8106-9768","affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100600436","display_name":"Lin Wang","orcid":"https://orcid.org/0000-0001-8353-2392"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang Lin","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-8353-2392","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101403129","display_name":"Jinyang Gao","orcid":"https://orcid.org/0000-0002-2094-3554"},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jinyang Gao","raw_affiliation_strings":["Alibaba Group, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-2094-3554","affiliations":[{"raw_affiliation_string":"Alibaba Group, Hangzhou, China","institution_ids":["https://openalex.org/I45928872"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041727387","display_name":"Kun Kuang","orcid":"https://orcid.org/0000-0001-7024-9790"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Kuang","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-7024-9790","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079260216","display_name":"Zhou Zhao","orcid":"https://orcid.org/0000-0001-6121-0384"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou Zhao","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-6121-0384","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004882141","display_name":"Fei Wu","orcid":"https://orcid.org/0000-0003-2139-8807"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Wu","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-2139-8807","affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3489","last_page":"3498"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.817363440990448},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6560776829719543},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5446732044219971}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.817363440990448},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6560776829719543},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5446732044219971}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681014","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681014","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W398859631","https://openalex.org/W1956340063","https://openalex.org/W2101105183","https://openalex.org/W2471768434","https://openalex.org/W2798959609","https://openalex.org/W2964196083","https://openalex.org/W2971157635","https://openalex.org/W2988981892","https://openalex.org/W3033161486","https://openalex.org/W3034682719","https://openalex.org/W4214478598","https://openalex.org/W4285265603","https://openalex.org/W4304092243","https://openalex.org/W4304092489","https://openalex.org/W4312933868","https://openalex.org/W4366850747","https://openalex.org/W4375869762","https://openalex.org/W4376167052","https://openalex.org/W4376312115","https://openalex.org/W4382318986","https://openalex.org/W4382322626","https://openalex.org/W4382460372","https://openalex.org/W4386057725","https://openalex.org/W4386076522","https://openalex.org/W4386794522","https://openalex.org/W4387294588","https://openalex.org/W4387968057","https://openalex.org/W4387968214","https://openalex.org/W4387968461","https://openalex.org/W4387969743","https://openalex.org/W4388186922","https://openalex.org/W4388294468","https://openalex.org/W4390041933","https://openalex.org/W4390722730","https://openalex.org/W4390873659","https://openalex.org/W4390874575","https://openalex.org/W4392019595","https://openalex.org/W4392736425"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Research":[0],"on":[1,213,245],"Multi-modal":[2,135],"Large":[3,61],"Language":[4,62],"Models":[5],"(MLLMs)":[6],"towards":[7],"the":[8,60,71,91,121,142,150,158,168,179,214,219,223,230],"multi-image":[9],"cross-modal":[10],"instruction":[11],"has":[12],"received":[13],"increasing":[14],"attention":[15],"and":[16,50,166,218,241],"made":[17],"significant":[18,117],"progress,":[19],"particularly":[20],"in":[21,38,65,82,90,114,149],"scenarios":[22,115],"involving":[23,141],"closely":[24],"resembling":[25],"images":[26,58,89,104,122,148,172],"(":[27],"e.g.,":[28],"change":[29],"captioning).":[30],"Existing":[31],"MLLMs":[32,199],"typically":[33],"follow":[34],"a":[35,96,184,234],"two-step":[36],"process":[37],"their":[39],"pipelines:":[40],"first,":[41],"extracting":[42],"visual":[43,54,75,124],"tokens":[44,55,76],"independently":[45],"for":[46,77,87,105,134,163,198,238,243],"each":[47,78],"input":[48],"image,":[49],"then":[51],"aligning":[52],"these":[53],"from":[56,194],"different":[57,83,88,147,171],"with":[59,207],"Model":[63],"(LLM)":[64],"its":[66],"textual":[67],"feature":[68],"space.":[69],"However,":[70],"independent":[72],"extraction":[73,152],"of":[74,98,100,160,170,190,225],"image":[79],"may":[80],"result":[81],"semantics":[84,169],"being":[85],"prioritized":[86],"first":[92],"step,":[93],"leading":[94],"to":[95,156],"lack":[97],"preservation":[99,159],"linking":[101,161],"information":[102,162],"among":[103,120],"subsequent":[106],"LLM":[107],"analysis.":[108],"This":[109],"issue":[110],"becomes":[111],"more":[112],"serious":[113],"where":[116],"variations":[118],"exist":[119],"(e.g.,":[123],"storytelling).":[125],"To":[126],"address":[127],"this":[128],"challenge,":[129],"we":[130,182],"introduce":[131],"Semantic":[132],"Alignment":[133],"large":[136,235],"language":[137],"models":[138],"(SAM).":[139],"By":[140],"bidirectional":[143],"semantic":[144],"guidance":[145],"between":[146],"visual-token":[151],"process,":[153],"SAM":[154,227],"aims":[155],"enhance":[157],"coherent":[164],"analysis":[165],"align":[167],"before":[173],"feeding":[174],"them":[175],"into":[176],"LLM.":[177],"As":[178],"test":[180],"bed,":[181],"propose":[183],"large-scale":[185],"dataset":[186,203],"named":[187],"MmLINK":[188,202],"consisting":[189],"69K":[191],"samples.":[192],"Different":[193],"most":[195],"existing":[196],"datasets":[197],"fine-tuning,":[200],"our":[201,226],"comprises":[204],"multi-modal":[205],"instructions":[206],"significantly":[208],"diverse":[209],"images.":[210],"Extensive":[211],"experiments":[212],"group":[215,239],"captioning":[216,240],"task":[217,221],"storytelling":[220,244],"prove":[222],"effectiveness":[224],"model,":[228],"surpassing":[229],"state-of-the-art":[231],"methods":[232],"by":[233],"margin":[236],"(+37%":[237],"+22%":[242],"CIDEr":[246],"score).":[247],"Project":[248],"page:":[249],"https://mccartney01.github.io/SAM.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":8}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}