{"id":"https://openalex.org/W4401070040","doi":"https://doi.org/10.1109/tcsvt.2024.3435561","title":"Asynchronous Multimodal Video Sequence Fusion via Learning Modality-Exclusive and -Agnostic Representations","display_name":"Asynchronous Multimodal Video Sequence Fusion via Learning Modality-Exclusive and -Agnostic Representations","publication_year":2024,"publication_date":"2024-07-29","ids":{"openalex":"https://openalex.org/W4401070040","doi":"https://doi.org/10.1109/tcsvt.2024.3435561"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2024.3435561","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3435561","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064280715","display_name":"Dingkang Yang","orcid":"https://orcid.org/0000-0003-1829-5671"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dingkang Yang","raw_affiliation_strings":["Academy for Engineering and Technology, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-1829-5671","affiliations":[{"raw_affiliation_string":"Academy for Engineering and Technology, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058367079","display_name":"M. H. Li","orcid":"https://orcid.org/0009-0000-6244-6081"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingcheng Li","raw_affiliation_strings":["Academy for Engineering and Technology, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0009-0000-6244-6081","affiliations":[{"raw_affiliation_string":"Academy for Engineering and Technology, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006193049","display_name":"Linhao Qu","orcid":"https://orcid.org/0000-0001-8815-7050"},"institutions":[{"id":"https://openalex.org/I133868755","display_name":"Shanghai Medical College of Fudan University","ror":"https://ror.org/01zntxs11","country_code":"CN","type":"education","lineage":["https://openalex.org/I133868755","https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linhao Qu","raw_affiliation_strings":["Digital Medical Research Center, School of Basic Medical Science, Fudan University, Shanghai, China","School of Basic Medical Science, Digital Medical Research Center, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0001-8815-7050","affiliations":[{"raw_affiliation_string":"Digital Medical Research Center, School of Basic Medical Science, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I133868755"]},{"raw_affiliation_string":"School of Basic Medical Science, Digital Medical Research Center, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I133868755"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100634445","display_name":"Kun Yang","orcid":"https://orcid.org/0000-0002-9956-2200"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kun Yang","raw_affiliation_strings":["Academy for Engineering and Technology, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9956-2200","affiliations":[{"raw_affiliation_string":"Academy for Engineering and Technology, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5036810000","display_name":"Peng Zhai","orcid":"https://orcid.org/0000-0002-1374-7969"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Zhai","raw_affiliation_strings":["Academy for Engineering and Technology, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-1374-7969","affiliations":[{"raw_affiliation_string":"Academy for Engineering and Technology, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082259804","display_name":"Song Wang","orcid":"https://orcid.org/0000-0003-4152-5295"},"institutions":[{"id":"https://openalex.org/I155781252","display_name":"University of South Carolina","ror":"https://ror.org/02b6qw903","country_code":"US","type":"education","lineage":["https://openalex.org/I155781252"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Song Wang","raw_affiliation_strings":["Department of Computer Science and Engineering, University of South Carolina, Columbia, SC, USA"],"raw_orcid":"https://orcid.org/0000-0003-4152-5295","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University of South Carolina, Columbia, SC, USA","institution_ids":["https://openalex.org/I155781252"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100414906","display_name":"Lihua Zhang","orcid":"https://orcid.org/0000-0003-0467-4347"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lihua Zhang","raw_affiliation_strings":["Academy for Engineering and Technology, Fudan University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-0467-4347","affiliations":[{"raw_affiliation_string":"Academy for Engineering and Technology, Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.2481,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.8916983,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"34","issue":"12","first_page":"12360","last_page":"12375"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9599000215530396,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9599000215530396,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.9592999815940857,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.9501000046730042,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/asynchronous-communication","display_name":"Asynchronous communication","score":0.7294349074363708},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7138630747795105},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.6131277680397034},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.571835458278656},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.49580296874046326},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.45760512351989746},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.4096762239933014},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.32883352041244507},{"id":"https://openalex.org/keywords/computer-network","display_name":"Computer network","score":0.14306065440177917}],"concepts":[{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.7294349074363708},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7138630747795105},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6131277680397034},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.571835458278656},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49580296874046326},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.45760512351989746},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4096762239933014},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.32883352041244507},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.14306065440177917},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2024.3435561","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3435561","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5901113696","display_name":null,"funder_award_id":"2021ZD0113503","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"},{"id":"https://openalex.org/G5970276311","display_name":null,"funder_award_id":"2021SHZDZX0103","funder_id":"https://openalex.org/F4320335480","funder_display_name":"Guangzhou Municipal Science and Technology Project"}],"funders":[{"id":"https://openalex.org/F4320335480","display_name":"Guangzhou Municipal Science and Technology Project","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":68,"referenced_works":["https://openalex.org/W2064675550","https://openalex.org/W2095176743","https://openalex.org/W2101267652","https://openalex.org/W2127141656","https://openalex.org/W2146334809","https://openalex.org/W2250539671","https://openalex.org/W2395639500","https://openalex.org/W2556418146","https://openalex.org/W2883409523","https://openalex.org/W2886193235","https://openalex.org/W2962931510","https://openalex.org/W2964051877","https://openalex.org/W2964216663","https://openalex.org/W2964346351","https://openalex.org/W2981458089","https://openalex.org/W2999634272","https://openalex.org/W3034266838","https://openalex.org/W3037572520","https://openalex.org/W3039263585","https://openalex.org/W3093051361","https://openalex.org/W3116109061","https://openalex.org/W3128412859","https://openalex.org/W3153682083","https://openalex.org/W3169801598","https://openalex.org/W3173549566","https://openalex.org/W3194484777","https://openalex.org/W3200823604","https://openalex.org/W3205519684","https://openalex.org/W3209114384","https://openalex.org/W3214432797","https://openalex.org/W4221163935","https://openalex.org/W4225694859","https://openalex.org/W4296437558","https://openalex.org/W4304092664","https://openalex.org/W4304098327","https://openalex.org/W4312377805","https://openalex.org/W4312497610","https://openalex.org/W4312609734","https://openalex.org/W4319460305","https://openalex.org/W4321608108","https://openalex.org/W4376607994","https://openalex.org/W4384519194","https://openalex.org/W4385245566","https://openalex.org/W4385570713","https://openalex.org/W4386075843","https://openalex.org/W4386075879","https://openalex.org/W4386699383","https://openalex.org/W4386799843","https://openalex.org/W4387609067","https://openalex.org/W4390889756","https://openalex.org/W4391468670","https://openalex.org/W4391488898","https://openalex.org/W4391614403","https://openalex.org/W4393147709","https://openalex.org/W4393970287","https://openalex.org/W4399343599","https://openalex.org/W4402713102","https://openalex.org/W4404025009","https://openalex.org/W6631190155","https://openalex.org/W6639480849","https://openalex.org/W6725448924","https://openalex.org/W6731132879","https://openalex.org/W6752724743","https://openalex.org/W6755977528","https://openalex.org/W6764072591","https://openalex.org/W6852678209","https://openalex.org/W6862919896","https://openalex.org/W6868834999"],"related_works":["https://openalex.org/W2116677773","https://openalex.org/W2155261584","https://openalex.org/W2385859805","https://openalex.org/W2584231425","https://openalex.org/W2530972254","https://openalex.org/W2150611273","https://openalex.org/W4207086172","https://openalex.org/W2042919702","https://openalex.org/W4225981436","https://openalex.org/W2354489033"],"abstract_inverted_index":{"Understanding":[0],"human":[1],"intentions":[2],"(e.g.,":[3],"emotions)":[4],"from":[5,22],"videos":[6],"has":[7],"received":[8],"considerable":[9],"attention":[10,120],"recently.":[11],"Video":[12],"streams":[13],"generally":[14],"constitute":[15],"a":[16,66,94,117,136,155],"blend":[17],"of":[18,37,145,191],"temporal":[19,45],"data":[20],"stemming":[21],"distinct":[23,86,146],"modalities,":[24],"including":[25],"natural":[26],"language,":[27],"facial":[28],"expressions,":[29],"and":[30,47,73,81,105,167],"auditory":[31],"clues.":[32],"Despite":[33],"the":[34,43,83,89,110,114,132,143,189],"impressive":[35],"advancements":[36],"previous":[38],"works":[39],"via":[40],"attention-based":[41],"paradigms,":[42],"inherent":[44],"asynchrony":[46],"modality":[48],"heterogeneity":[49],"challenges":[50],"remain":[51],"in":[52,148],"multimodal":[53,79,170,181],"sequence":[54],"fusion,":[55],"causing":[56],"adverse":[57],"performance":[58],"bottlenecks.":[59],"To":[60],"tackle":[61],"these":[62],"issues,":[63],"we":[64,153],"propose":[65,154],"Multimodal":[67],"fusion":[68,158],"approach":[69],"for":[70,172],"learning":[71],"modality-Exclusive":[72],"modality-Agnostic":[74],"representations":[75,147,171],"(MEA)":[76],"to":[77,98,124,141,160],"refine":[78],"features":[80,108],"leverage":[82],"complementarity":[84],"across":[85,164],"modalities.":[87],"On":[88,113],"one":[90],"hand,":[91,116],"MEA":[92],"introduces":[93],"predictive":[95],"self-attention":[96],"module":[97,121],"capture":[99],"reliable":[100],"context":[101],"dynamics":[102],"within":[103],"modalities":[104,130,166],"reinforce":[106],"unique":[107],"over":[109,131],"modality-exclusive":[111],"spaces.":[112],"other":[115],"hierarchical":[118],"cross-modal":[119],"is":[122,139],"designed":[123],"explore":[125],"valuable":[126],"element":[127],"correlations":[128],"among":[129],"modality-agnostic":[133],"space.":[134],"Meanwhile,":[135],"double-discriminator":[137],"strategy":[138],"presented":[140],"ensure":[142],"production":[144],"an":[149],"adversarial":[150],"manner.":[151],"Eventually,":[152],"decoupled":[156],"graph":[157],"mechanism":[159],"enhance":[161],"knowledge":[162],"exchange":[163],"heterogeneous":[165],"learn":[168],"robust":[169],"downstream":[173],"tasks.":[174],"Numerous":[175],"experiments":[176],"are":[177],"implemented":[178],"on":[179],"three":[180],"datasets":[182],"with":[183],"asynchronous":[184],"sequences.":[185],"Systematic":[186],"analyses":[187],"show":[188],"necessity":[190],"our":[192],"approach.":[193]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":8}],"updated_date":"2025-12-26T23:08:49.675405","created_date":"2025-10-10T00:00:00"}