{"id":"https://openalex.org/W7165666335","doi":"https://doi.org/10.48550/arxiv.2606.21197","title":"Extraction and Analysis of Multimodal Concepts in Vision Language Models through Sparse Autoencoders","display_name":"Extraction and Analysis of Multimodal Concepts in Vision Language Models through Sparse Autoencoders","publication_year":2026,"publication_date":"2026-06-19","ids":{"openalex":"https://openalex.org/W7165666335","doi":"https://doi.org/10.48550/arxiv.2606.21197"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.21197","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21197","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.21197","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113027759","display_name":"Sergio Lanza","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lanza, Sergio","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139135582","display_name":"Jae Hee Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Jae Hee","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139209502","display_name":"Stefan Wermter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wermter, Stefan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11148","display_name":"Language, Metaphor, and Cognition","score":0.0010999999940395355,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.6658999919891357},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5185999870300293},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.5160999894142151},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.4205999970436096},{"id":"https://openalex.org/keywords/cosine-similarity","display_name":"Cosine similarity","score":0.39879998564720154},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.38999998569488525},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.38999998569488525},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.387800008058548},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.3806000053882599}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8012999892234802},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.6658999919891357},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6179999709129333},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5185999870300293},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.5160999894142151},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43849998712539673},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.4205999970436096},{"id":"https://openalex.org/C2780762811","wikidata":"https://www.wikidata.org/wiki/Q1784941","display_name":"Cosine similarity","level":3,"score":0.39879998564720154},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3946000039577484},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.38999998569488525},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.38999998569488525},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.387800008058548},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.3806000053882599},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.36230000853538513},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.34389999508857727},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.34360000491142273},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.3314000070095062},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.3192000091075897},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.2964000105857849},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.2955000102519989},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.2702000141143799},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C197947376","wikidata":"https://www.wikidata.org/wiki/Q5155608","display_name":"Comparability","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26089999079704285},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.259799987077713},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.25369998812675476},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.25209999084472656}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.21197","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21197","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.21197","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21197","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.716801643371582}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Vision":[0],"Language":[1],"Models":[2],"(VLMs)":[3],"have":[4,39],"demonstrated":[5],"impressive":[6],"performance":[7],"in":[8,52],"tasks":[9],"requiring":[10],"joint":[11],"understanding":[12,28,74,105],"of":[13,29,49,75,183,195],"images":[14],"and":[15,21,117,121,135,142,179,206],"text,":[16],"such":[17],"as":[18,41],"image":[19],"captioning":[20],"Visual":[22],"Question":[23],"Answering":[24],"(VQA),":[25],"but":[26],"our":[27,158],"their":[30,102],"internal":[31],"processes":[32],"remains":[33],"limited.":[34],"Recently,":[35],"Sparse":[36],"Autoencoders":[37],"(SAEs)":[38],"emerged":[40],"a":[42,72,110,131,152,198],"promising":[43],"tool":[44],"to":[45,115,166,169,201],"support":[46],"the":[47,137,140,143,192],"interpretation":[48],"concepts":[50,64,78,123],"encoded":[51],"VLMs.":[53,125],"However,":[54],"most":[55],"SAE-based":[56,171],"approaches":[57,89],"focus":[58],"only":[59],"on":[60,113,151],"textual":[61,176],"or":[62,99],"visual":[63,88,93,161],"separately,":[65],"ignoring":[66],"multimodal":[67,122,184,207],"concepts.":[68,185,208],"This":[69,186],"limitation":[70],"hinders":[71],"comprehensive":[73],"VLMs,":[76,196],"since":[77],"that":[79,96,157],"integrate":[80],"both":[81],"modalities":[82],"can":[83],"be":[84],"misclassified.":[85],"Moreover,":[86],"previous":[87],"often":[90],"produce":[91],"low-quality":[92],"concept":[94,134,141,162,177],"descriptions":[95],"are":[97],"vague":[98],"incomplete,":[100],"limiting":[101],"usefulness":[103],"for":[104],"model":[106],"reasoning.":[107],"We":[108],"propose":[109,130],"framework":[111,159],"based":[112],"SAEs":[114],"extract":[116],"analyze":[118],"visual,":[119,204],"textual,":[120,205],"from":[124],"For":[126],"each":[127],"neuron,":[128],"we":[129],"candidate":[132],"human-interpretable":[133],"compute":[136],"alignment":[138],"between":[139,203],"dataset":[144,154],"samples":[145],"using":[146],"cosine":[147],"similarity":[148],"scores.":[149],"Experiments":[150],"VQA":[153],"(LLaVA-NeXT)":[155],"demonstrate":[156],"improves":[160],"quality":[163,178],"by":[164],"up":[165],"45\\%":[167],"compared":[168],"existing":[170],"methods,":[172],"while":[173],"maintaining":[174],"high":[175],"enabling":[180],"systematic":[181],"identification":[182],"work":[187],"contributes":[188],"new":[189],"insights":[190],"into":[191],"conceptual":[193],"space":[194],"providing":[197],"structured":[199],"approach":[200],"distinguish":[202],"The":[209],"code":[210],"is":[211],"available":[212],"at":[213],"https://github.com/PHDLanza/Multidata_SAE":[214]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-24T00:00:00"}