{"id":"https://openalex.org/W4398152820","doi":"https://doi.org/10.1109/tcsvt.2024.3403167","title":"Cross-Modality Image Interpretation via Concept Decomposition Vector of Visual-Language Models","display_name":"Cross-Modality Image Interpretation via Concept Decomposition Vector of Visual-Language Models","publication_year":2024,"publication_date":"2024-05-20","ids":{"openalex":"https://openalex.org/W4398152820","doi":"https://doi.org/10.1109/tcsvt.2024.3403167"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2024.3403167","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3403167","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054512186","display_name":"Zhengqing Fang","orcid":"https://orcid.org/0000-0003-3270-0952"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]},{"id":"https://openalex.org/I2802390878","display_name":"Sir Run Run Shaw Hospital","ror":"https://ror.org/00ka6rp58","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2802390878"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengqing Fang","raw_affiliation_strings":["Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","College of Computer Science and Technology, Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-3270-0952","affiliations":[{"raw_affiliation_string":"Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","institution_ids":["https://openalex.org/I2802390878","https://openalex.org/I76130692"]},{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113379229","display_name":"Zhouhang Yuan","orcid":null},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]},{"id":"https://openalex.org/I2802390878","display_name":"Sir Run Run Shaw Hospital","ror":"https://ror.org/00ka6rp58","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2802390878"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhouhang Yuan","raw_affiliation_strings":["Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","College of Computer Science and Technology, Zhejiang University, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","institution_ids":["https://openalex.org/I2802390878","https://openalex.org/I76130692"]},{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102673227","display_name":"Ziyu Li","orcid":"https://orcid.org/0009-0007-2536-6278"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyu Li","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jingyuan Chen","orcid":"https://orcid.org/0000-0003-0415-6937"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jingyuan Chen","raw_affiliation_strings":["College of Education, Zhejiang University, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-0415-6937","affiliations":[{"raw_affiliation_string":"College of Education, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041727387","display_name":"Kun Kuang","orcid":"https://orcid.org/0000-0001-7024-9790"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Kuang","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou, China","Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-7024-9790","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]},{"raw_affiliation_string":"Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101966833","display_name":"Yu\u2010Feng Yao","orcid":"https://orcid.org/0000-0001-5699-3388"},"institutions":[{"id":"https://openalex.org/I2802390878","display_name":"Sir Run Run Shaw Hospital","ror":"https://ror.org/00ka6rp58","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I2802390878"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu-Feng Yao","raw_affiliation_strings":["Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Ophthalmology, Sir Run Run Shaw Hospital, Zhejiang University School of Medicine, Hangzhou, China","institution_ids":["https://openalex.org/I2802390878","https://openalex.org/I76130692"]},{"raw_affiliation_string":"Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004882141","display_name":"Fei Wu","orcid":"https://orcid.org/0000-0003-2139-8807"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fei Wu","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou, China","Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-2139-8807","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]},{"raw_affiliation_string":"Key Laboratory for Corneal Diseases Research of Zhejiang Province, Hangzhou, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6131,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.65848645,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"35","issue":"4","first_page":"3024","last_page":"3038"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9659000039100647,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9659000039100647,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9573000073432922,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9484999775886536,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5957629084587097},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5916817784309387},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5824418663978577},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.5396132469177246},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.5051494240760803},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.4729085862636566},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.41984444856643677},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3775727450847626},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1873222291469574},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.0902164876461029}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5957629084587097},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5916817784309387},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5824418663978577},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.5396132469177246},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.5051494240760803},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4729085862636566},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41984444856643677},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3775727450847626},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1873222291469574},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0902164876461029},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2024.3403167","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3403167","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.4099999964237213}],"awards":[{"id":"https://openalex.org/G3253937777","display_name":null,"funder_award_id":"62376243","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4220270284","display_name":null,"funder_award_id":"62307032","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4385699040","display_name":null,"funder_award_id":"U20A20387","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6619237417","display_name":null,"funder_award_id":"2022ZD0118003","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G6673228248","display_name":null,"funder_award_id":"62307032","funder_id":"https://openalex.org/F4320322678","funder_display_name":"National Youth Foundation of China"},{"id":"https://openalex.org/G6824276054","display_name":null,"funder_award_id":"U20A20387","funder_id":"https://openalex.org/F4320334064","funder_display_name":"National Natural Science Foundation of China-Zhejiang Joint Fund for the Integration of Industrialization and Informatization"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322678","display_name":"National Youth Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334064","display_name":"National Natural Science Foundation of China-Zhejiang Joint Fund for the Integration of Industrialization and Informatization","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":60,"referenced_works":["https://openalex.org/W2047643928","https://openalex.org/W2516809705","https://openalex.org/W2794825826","https://openalex.org/W2945976633","https://openalex.org/W2962858109","https://openalex.org/W2964194231","https://openalex.org/W2974979868","https://openalex.org/W3012380364","https://openalex.org/W3092798752","https://openalex.org/W3212509912","https://openalex.org/W3215928733","https://openalex.org/W4292805535","https://openalex.org/W4297697565","https://openalex.org/W4313156423","https://openalex.org/W4321786089","https://openalex.org/W4362471316","https://openalex.org/W4385763854","https://openalex.org/W4386065251","https://openalex.org/W4386075969","https://openalex.org/W4386076063","https://openalex.org/W4390023570","https://openalex.org/W4390190100","https://openalex.org/W4390874575","https://openalex.org/W4392172801","https://openalex.org/W4393150390","https://openalex.org/W4394758452","https://openalex.org/W6638677478","https://openalex.org/W6750391026","https://openalex.org/W6753001334","https://openalex.org/W6761184903","https://openalex.org/W6764462969","https://openalex.org/W6778883912","https://openalex.org/W6779243538","https://openalex.org/W6786269015","https://openalex.org/W6789753369","https://openalex.org/W6791353385","https://openalex.org/W6795562474","https://openalex.org/W6796524997","https://openalex.org/W6796581206","https://openalex.org/W6797698596","https://openalex.org/W6801948084","https://openalex.org/W6803476161","https://openalex.org/W6810452849","https://openalex.org/W6810940779","https://openalex.org/W6838961916","https://openalex.org/W6839745749","https://openalex.org/W6842358254","https://openalex.org/W6842947093","https://openalex.org/W6843355076","https://openalex.org/W6847300921","https://openalex.org/W6849177959","https://openalex.org/W6849654975","https://openalex.org/W6849976536","https://openalex.org/W6850730005","https://openalex.org/W6851326247","https://openalex.org/W6852577244","https://openalex.org/W6854059577","https://openalex.org/W6854989695","https://openalex.org/W6857802407","https://openalex.org/W6860061640"],"related_works":["https://openalex.org/W4313320911","https://openalex.org/W2385859805","https://openalex.org/W4327743144","https://openalex.org/W2530972254","https://openalex.org/W4245077728","https://openalex.org/W2607424049","https://openalex.org/W4390922876","https://openalex.org/W3183204001","https://openalex.org/W4206302830","https://openalex.org/W2185941092"],"abstract_inverted_index":{"Interpretable":[0],"image":[1,50,95],"classification":[2,196],"is":[3,60,128],"crucial":[4],"for":[5,222],"making":[6],"decisions":[7],"in":[8,94,167,195,202,220,226],"high-stakes":[9],"scenarios.":[10],"Recent":[11],"advancements":[12],"have":[13],"demonstrated":[14],"that":[15,113,182],"interpretable":[16],"models":[17,24,37],"can":[18,139],"achieve":[19],"performance":[20,186],"comparable":[21],"to":[22,188,206,228],"black-box":[23],"by":[25,41,231],"integrating":[26],"Visual":[27],"Language":[28],"Models":[29,34],"(VLMs)":[30],"with":[31,175,191],"Concept":[32],"Bottleneck":[33],"(CBMs).":[35],"These":[36],"explain":[38],"their":[39],"predictions":[40],"calculating":[42],"the":[43,49,71,88,162,170,213],"weighted":[44],"sum":[45],"of":[46,73,91,109,120,126,153,164,215],"similarities":[47],"between":[48],"representation":[51],"and":[52,62,79,145,155,198],"predefined":[53,121],"text":[54,122],"embeddings.":[55],"However,":[56],"selecting":[57],"textual":[58,66,144],"descriptors":[59],"subjective,":[61],"relying":[63],"solely":[64],"on":[65],"information":[67],"may":[68],"not":[69],"capture":[70],"complexities":[72],"visual":[74,111,146,224],"data,":[75],"impacting":[76],"both":[77,143],"interpretability":[78],"performance.":[80],"To":[81],"address":[82],"these":[83],"limitations,":[84],"this":[85],"work":[86],"explores":[87],"cross-modality":[89],"interpretation":[90],"class-related":[92],"concepts":[93,112,127,154,166],"classification.":[96],"Specifically,":[97],"we":[98,211],"propose":[99],"decomposed":[100,110,165],"concept":[101,134],"bottleneck":[102],"model":[103],"(DCBM),":[104],"which":[105,138,160],"utilizes":[106],"a":[107,150,156,192,199],"set":[108],"are":[114],"extracted":[115],"directly":[116],"from":[117],"images":[118,168],"instead":[119],"concepts.":[123],"The":[124],"decomposition":[125,135],"achieved":[129],"through":[130],"vector":[131],"projection":[132],"onto":[133],"vectors":[136],"(CDVs),":[137],"be":[140],"interpreted":[141],"across":[142],"modalities.":[147],"We":[148],"introduce":[149],"quintuple":[151],"notion":[152],"concept-sample":[157],"distribution":[158],"theorem,":[159],"enables":[161],"localization":[163],"using":[169],"Segment":[171],"Anything":[172],"Model":[173],"(SAM)":[174],"automatically":[176,217],"generated":[177,218],"prompts.":[178],"Experimental":[179],"results":[180],"demonstrate":[181],"DCBM":[183],"achieves":[184],"competitive":[185],"compared":[187,205],"non-interpretable":[189],"models,":[190],"3.42%":[193],"improvement":[194,201],"accuracy":[197],"66.27%":[200],"image-text":[203],"groundability":[204],"other":[207],"VLM-based":[208],"CBMs.":[209],"Furthermore,":[210],"evaluate":[212],"benefits":[214],"employing":[216],"prompts":[219,229],"SAM":[221],"interpreting":[223],"concepts,":[225],"contrast":[227],"created":[230],"human":[232],"operators.":[233]},"counts_by_year":[{"year":2025,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}