{"id":"https://openalex.org/W4412722424","doi":"https://doi.org/10.1109/tce.2025.3593333","title":"Learning Three-Modal Joint Representations With Multimodal Transformer for Multimodal Sentiment Analysis","display_name":"Learning Three-Modal Joint Representations With Multimodal Transformer for Multimodal Sentiment Analysis","publication_year":2025,"publication_date":"2025-07-29","ids":{"openalex":"https://openalex.org/W4412722424","doi":"https://doi.org/10.1109/tce.2025.3593333"},"language":"en","primary_location":{"id":"doi:10.1109/tce.2025.3593333","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tce.2025.3593333","pdf_url":null,"source":{"id":"https://openalex.org/S126824455","display_name":"IEEE Transactions on Consumer Electronics","issn_l":"0098-3063","issn":["0098-3063","1558-4127"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Consumer Electronics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053626221","display_name":"Kuijie Zhang","orcid":"https://orcid.org/0000-0003-1036-6893"},"institutions":[{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Kuijie Zhang","raw_affiliation_strings":["State Key Laboratory of Chemical Safety, Shandong Key Laboratory of Intelligent Oil & Gas Industrial Software, and the Qingdao Institute of Software, China University of Petroleum, Qingdao, China","Shandong Key Laboratory of Intelligent Oil and Gas Industrial Software, the Qingdao Institute of Software, State Key Laboratory of Chemical Safety, China University of Petroleum, Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0003-1036-6893","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Chemical Safety, Shandong Key Laboratory of Intelligent Oil & Gas Industrial Software, and the Qingdao Institute of Software, China University of Petroleum, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]},{"raw_affiliation_string":"Shandong Key Laboratory of Intelligent Oil and Gas Industrial Software, the Qingdao Institute of Software, State Key Laboratory of Chemical Safety, China University of Petroleum, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hongjuan Pei","orcid":null},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongjuan Pei","raw_affiliation_strings":["School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Engineering Science, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068648599","display_name":"Chenkai Zhang","orcid":"https://orcid.org/0009-0004-1583-0506"},"institutions":[{"id":"https://openalex.org/I43337087","display_name":"Hebei University","ror":"https://ror.org/01p884a79","country_code":"CN","type":"education","lineage":["https://openalex.org/I43337087"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenkai Zhang","raw_affiliation_strings":["College of Mathematics and Information Science, Hebei University, Baoding, Hebei, China","College of Mathematics and Information Science, Hebei University, Hebei, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Mathematics and Information Science, Hebei University, Baoding, Hebei, China","institution_ids":["https://openalex.org/I43337087"]},{"raw_affiliation_string":"College of Mathematics and Information Science, Hebei University, Hebei, China","institution_ids":["https://openalex.org/I43337087"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5054060458","display_name":"Shanchen Pang","orcid":"https://orcid.org/0000-0002-5705-1218"},"institutions":[{"id":"https://openalex.org/I4210162190","display_name":"China University of Petroleum, East China","ror":"https://ror.org/05gbn2817","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210162190"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shanchen Pang","raw_affiliation_strings":["State Key Laboratory of Chemical Safety, Shandong Key Laboratory of Intelligent Oil & Gas Industrial Software, and the Qingdao Institute of Software, China University of Petroleum, Qingdao, China","Shandong Key Laboratory of Intelligent Oil and Gas Industrial Software, the Qingdao Institute of Software, State Key Laboratory of Chemical Safety, China University of Petroleum, Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0002-5705-1218","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Chemical Safety, Shandong Key Laboratory of Intelligent Oil & Gas Industrial Software, and the Qingdao Institute of Software, China University of Petroleum, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]},{"raw_affiliation_string":"Shandong Key Laboratory of Intelligent Oil and Gas Industrial Software, the Qingdao Institute of Software, State Key Laboratory of Chemical Safety, China University of Petroleum, Qingdao, China","institution_ids":["https://openalex.org/I4210162190"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5053626221"],"corresponding_institution_ids":["https://openalex.org/I4210162190"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0875424,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"71","issue":"3","first_page":"7559","last_page":"7569"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9876000285148621,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9003000259399414,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5888764262199402},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.5665059685707092},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.5526936650276184},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.535463273525238},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5043414831161499},{"id":"https://openalex.org/keywords/multimodal-therapy","display_name":"Multimodal therapy","score":0.48480239510536194},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.46031802892684937},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.4224269390106201},{"id":"https://openalex.org/keywords/multimodal-interaction","display_name":"Multimodal interaction","score":0.41754305362701416},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.40092113614082336},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3608224391937256},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.27250754833221436},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.22350844740867615},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.16459092497825623},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.15452778339385986},{"id":"https://openalex.org/keywords/structural-engineering","display_name":"Structural engineering","score":0.13141492009162903},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09617465734481812},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.08183956146240234}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5888764262199402},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.5665059685707092},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.5526936650276184},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.535463273525238},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5043414831161499},{"id":"https://openalex.org/C4441509","wikidata":"https://www.wikidata.org/wiki/Q6418787","display_name":"Multimodal therapy","level":2,"score":0.48480239510536194},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.46031802892684937},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.4224269390106201},{"id":"https://openalex.org/C135641252","wikidata":"https://www.wikidata.org/wiki/Q738567","display_name":"Multimodal interaction","level":2,"score":0.41754305362701416},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40092113614082336},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3608224391937256},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.27250754833221436},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.22350844740867615},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.16459092497825623},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.15452778339385986},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.13141492009162903},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09617465734481812},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.08183956146240234},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tce.2025.3593333","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tce.2025.3593333","pdf_url":null,"source":{"id":"https://openalex.org/S126824455","display_name":"IEEE Transactions on Consumer Electronics","issn_l":"0098-3063","issn":["0098-3063","1558-4127"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Consumer Electronics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4423525907","display_name":null,"funder_award_id":"ZR2024MF129","funder_id":"https://openalex.org/F4320324174","funder_display_name":"Natural Science Foundation of Shandong Province"},{"id":"https://openalex.org/G7895523656","display_name":null,"funder_award_id":"2021YFA1000102","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320324174","display_name":"Natural Science Foundation of Shandong Province","ror":null},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W2064675550","https://openalex.org/W2787581402","https://openalex.org/W2883409523","https://openalex.org/W2962931510","https://openalex.org/W2964010806","https://openalex.org/W2964346351","https://openalex.org/W3093051361","https://openalex.org/W4362579348","https://openalex.org/W4389352393","https://openalex.org/W4390659280","https://openalex.org/W4390813148","https://openalex.org/W4391128766","https://openalex.org/W4391344842","https://openalex.org/W4393227307","https://openalex.org/W4396861023","https://openalex.org/W4398765351","https://openalex.org/W4400191939","https://openalex.org/W4400927401","https://openalex.org/W4401044014","https://openalex.org/W4401288968","https://openalex.org/W4401332426","https://openalex.org/W4401615828","https://openalex.org/W4402455114","https://openalex.org/W4402727922","https://openalex.org/W4404875427","https://openalex.org/W4404876522","https://openalex.org/W4415302351"],"related_works":["https://openalex.org/W2968820833","https://openalex.org/W4380355808","https://openalex.org/W3093803775","https://openalex.org/W2068964259","https://openalex.org/W2013013717","https://openalex.org/W2558617677","https://openalex.org/W2342456189","https://openalex.org/W66256988","https://openalex.org/W4302857920","https://openalex.org/W4406273444"],"abstract_inverted_index":{"Understanding":[0],"user":[1,128],"sentiment":[2,27],"from":[3],"multiple":[4,54],"modalities":[5],"is":[6],"essential":[7],"for":[8,25,123],"applications":[9],"such":[10],"as":[11],"personalized":[12],"recommendations":[13],"and":[14,41,63,75,87,95,101,111,126],"affective":[15],"computing.":[16],"We":[17,67],"propose":[18],"TMJR,":[19],"a":[20,31,88,102,120],"Three-modal":[21],"Joint":[22],"Representation":[23],"model":[24],"multimodal":[26,127],"analysis,":[28],"which":[29],"introduces":[30],"structured":[32],"three-round":[33],"cross-modal":[34,61],"attention":[35],"mechanism":[36],"to":[37,58],"integrate":[38],"textual,":[39],"acoustic,":[40],"visual":[42],"information":[43],"progressively.":[44],"Unlike":[45],"prior":[46],"one-pass":[47],"or":[48],"pairwise":[49],"fusion":[50],"methods,":[51],"TMJR":[52,69,118],"models":[53],"directional":[55],"interaction":[56],"paths":[57],"capture":[59],"deeper":[60],"dependencies":[62],"improve":[64],"semantic":[65],"alignment.":[66],"evaluate":[68],"on":[70,93,106],"two":[71],"benchmark":[72],"datasets:":[73],"CMU-MOSI":[74],"CMU-MOSEI.":[76],"It":[77],"achieves":[78],"state-of-the-art":[79],"results,":[80],"with":[81],"an":[82,96],"F1":[83,97],"score":[84,98],"of":[85,91,99,104],"86.4%":[86],"Pearson":[89],"correlation":[90,103],"0.799":[92],"MOSI,":[94],"86.5%":[100],"0.775":[105],"MOSEI.":[107],"Further":[108],"ablation,":[109],"label-efficiency,":[110],"visualization":[112],"analyses":[113],"confirm":[114],"the":[115],"model\u2019s":[116],"robustness.":[117],"provides":[119],"promising":[121],"solution":[122],"sentiment-aware":[124],"recommendation":[125],"understanding.":[129]},"counts_by_year":[],"updated_date":"2025-11-12T23:11:45.498971","created_date":"2025-10-10T00:00:00"}