{"id":"https://openalex.org/W4402351708","doi":"https://doi.org/10.1109/tvcg.2024.3456387","title":"ModalChorus: Visual Probing and Alignment of Multi-Modal Embeddings via Modal Fusion Map","display_name":"ModalChorus: Visual Probing and Alignment of Multi-Modal Embeddings via Modal Fusion Map","publication_year":2024,"publication_date":"2024-09-09","ids":{"openalex":"https://openalex.org/W4402351708","doi":"https://doi.org/10.1109/tvcg.2024.3456387","pmid":"https://pubmed.ncbi.nlm.nih.gov/39250410"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2024.3456387","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2024.3456387","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051941811","display_name":"Yilin Ye","orcid":"https://orcid.org/0000-0001-8874-5928"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yilin Ye","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong","Hong Kong University of Science and Technology (Guangzhou), Hong Kong"],"raw_orcid":"https://orcid.org/0000-0001-8874-5928","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Hong Kong","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075939839","display_name":"Shishi Xiao","orcid":"https://orcid.org/0009-0008-0262-5289"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shishi Xiao","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Hong Kong","Hong Kong University of Science and Technology (Guangzhou), Hong Kong"],"raw_orcid":"https://orcid.org/0009-0008-0262-5289","affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Hong Kong","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Hong Kong","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087805627","display_name":"Xingchen Zeng","orcid":"https://orcid.org/0000-0002-1967-0595"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Xingchen Zeng","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Hong Kong","Hong Kong University of Science and Technology (Guangzhou), Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Guangzhou, Hong Kong","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Hong Kong","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100660205","display_name":"Wei Zeng","orcid":"https://orcid.org/0000-0002-5600-8824"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wei Zeng","raw_affiliation_strings":["Hong Kong University of Science and Technology, Hong Kong","Hong Kong University of Science and Technology (Guangzhou), Hong Kong"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology, Hong Kong","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), Hong Kong","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3124,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.8164523,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"31","issue":"1","first_page":"294","last_page":"304"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12111","display_name":"Industrial Vision Systems and Defect Detection","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/2209","display_name":"Industrial and Manufacturing Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7802095413208008},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7639185190200806},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.5608514547348022},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.5425747632980347},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.518881618976593},{"id":"https://openalex.org/keywords/computer-graphics","display_name":"Computer graphics (images)","score":0.5128036141395569},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.5031771063804626},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.49273374676704407},{"id":"https://openalex.org/keywords/data-visualization","display_name":"Data visualization","score":0.4314251244068146}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7802095413208008},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7639185190200806},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5608514547348022},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.5425747632980347},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.518881618976593},{"id":"https://openalex.org/C121684516","wikidata":"https://www.wikidata.org/wiki/Q7600677","display_name":"Computer graphics (images)","level":1,"score":0.5128036141395569},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.5031771063804626},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.49273374676704407},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.4314251244068146},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tvcg.2024.3456387","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2024.3456387","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:39250410","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/39250410","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-147212","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-147212","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4023314675","display_name":null,"funder_award_id":"62172398","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":75,"referenced_works":["https://openalex.org/W1861492603","https://openalex.org/W1893905004","https://openalex.org/W1956340063","https://openalex.org/W2037133710","https://openalex.org/W2089344418","https://openalex.org/W2089468765","https://openalex.org/W2096733369","https://openalex.org/W2100437550","https://openalex.org/W2187874575","https://openalex.org/W2203556438","https://openalex.org/W2506483933","https://openalex.org/W2560269799","https://openalex.org/W2751627669","https://openalex.org/W2769280657","https://openalex.org/W2771056734","https://openalex.org/W2789251334","https://openalex.org/W2803808737","https://openalex.org/W2869198903","https://openalex.org/W2958874938","https://openalex.org/W2986308225","https://openalex.org/W3094617751","https://openalex.org/W3096655658","https://openalex.org/W3151945894","https://openalex.org/W3153469116","https://openalex.org/W3177269615","https://openalex.org/W3202567403","https://openalex.org/W4200283487","https://openalex.org/W4225080353","https://openalex.org/W4225832925","https://openalex.org/W4226021591","https://openalex.org/W4235887661","https://openalex.org/W4292828970","https://openalex.org/W4297461125","https://openalex.org/W4301393026","https://openalex.org/W4309668639","https://openalex.org/W4312563428","https://openalex.org/W4312695444","https://openalex.org/W4312825288","https://openalex.org/W4312933868","https://openalex.org/W4366547425","https://openalex.org/W4381597284","https://openalex.org/W4385536252","https://openalex.org/W4385774603","https://openalex.org/W4386071707","https://openalex.org/W4386072096","https://openalex.org/W4386075490","https://openalex.org/W4386158887","https://openalex.org/W4386790226","https://openalex.org/W4387886076","https://openalex.org/W4387934929","https://openalex.org/W4390873530","https://openalex.org/W4390874575","https://openalex.org/W4391741748","https://openalex.org/W4393158799","https://openalex.org/W4396218624","https://openalex.org/W4396568157","https://openalex.org/W4396851234","https://openalex.org/W4399310710","https://openalex.org/W4402402086","https://openalex.org/W4402727091","https://openalex.org/W4402727368","https://openalex.org/W6674573128","https://openalex.org/W6674751278","https://openalex.org/W6682691769","https://openalex.org/W6691640376","https://openalex.org/W6766904570","https://openalex.org/W6774314701","https://openalex.org/W6790019176","https://openalex.org/W6791353385","https://openalex.org/W6796581206","https://openalex.org/W6809885388","https://openalex.org/W6810452849","https://openalex.org/W6841755765","https://openalex.org/W6849177959","https://openalex.org/W6860976756"],"related_works":["https://openalex.org/W2132659060","https://openalex.org/W2031992971","https://openalex.org/W3214791684","https://openalex.org/W2013728941","https://openalex.org/W2353265673","https://openalex.org/W4225274103","https://openalex.org/W2579659702","https://openalex.org/W2154046714","https://openalex.org/W2031175860","https://openalex.org/W1965329638"],"abstract_inverted_index":{"Multi-modal":[0],"embeddings":[1,20,109],"form":[2],"the":[3,12,127],"foundation":[4],"for":[5,47,97,107],"vision-language":[6,137],"models,":[7],"such":[8],"as":[9],"CLIP":[10,108],"embeddings,":[11],"most":[13],"widely":[14],"used":[15],"text-image":[16],"embeddings.":[17,54],"However,":[18],"these":[19],"are":[21],"vulnerable":[22],"to":[23,82,93,159],"subtle":[24],"misalignment":[25,149],"of":[26,52,129,148],"cross-modal":[27,133,160],"features,":[28],"resulting":[29],"in":[30,131,153],"decreased":[31],"model":[32],"performance":[33],"and":[34,50,79,86,100,104,116,118,150,162],"diminished":[35],"generalization.":[36],"To":[37],"address":[38],"this":[39],"problem,":[40],"we":[41],"design":[42],"ModalChorus,":[43],"an":[44],"interactive":[45],"system":[46],"visual":[48],"probing":[49,63],"alignment":[51,89],"multi-modal":[53],"ModalChorus":[55,143],"primarily":[56],"offers":[57],"a":[58,69],"two-stage":[59],"process:":[60],"1)":[61],"embedding":[62,88],"with":[64,110],"Modal":[65],"Fusion":[66],"Map":[67],"(MFM),":[68],"novel":[70],"parametric":[71],"dimensionality":[72,112],"reduction":[73,113],"method":[74],"that":[75,90,142],"integrates":[76],"both":[77,98],"metric":[78],"nonmetric":[80],"objectives":[81],"enhance":[83],"modality":[84],"fusion;":[85],"2)":[87],"allows":[91],"users":[92],"interactively":[94],"articulate":[95],"intentions":[96],"point-set":[99],"set-set":[101],"alignments.":[102],"Quantitative":[103],"qualitative":[105],"comparisons":[106],"existing":[111],"(e.g.,":[114,121],"t-SNE":[115],"MDS)":[117],"data":[119,122],"fusion":[120],"context":[123],"map)":[124],"methods":[125],"demonstrate":[126],"advantages":[128],"MFM":[130],"showcasing":[132],"features":[134],"over":[135],"common":[136],"datasets.":[138],"Case":[139],"studies":[140],"reveal":[141],"can":[144],"facilitate":[145],"intuitive":[146],"discovery":[147],"efficient":[151],"re-alignment":[152],"scenarios":[154],"ranging":[155],"from":[156],"zero-shot":[157],"classification":[158],"retrieval":[161],"generation.":[163]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}