{"id":"https://openalex.org/W4405936033","doi":"https://doi.org/10.1109/taffc.2024.3524418","title":"From EEG to Eye Movements: Cross-Modal Emotion Recognition Using Constrained Adversarial Network With Dual Attention","display_name":"From EEG to Eye Movements: Cross-Modal Emotion Recognition Using Constrained Adversarial Network With Dual Attention","publication_year":2024,"publication_date":"2024-12-31","ids":{"openalex":"https://openalex.org/W4405936033","doi":"https://doi.org/10.1109/taffc.2024.3524418"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2024.3524418","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2024.3524418","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100389973","display_name":"Yiting Wang","orcid":"https://orcid.org/0000-0003-3224-239X"},"institutions":[{"id":"https://openalex.org/I28166907","display_name":"Karolinska Institutet","ror":"https://ror.org/056d84691","country_code":"SE","type":"education","lineage":["https://openalex.org/I28166907"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Yiting Wang","raw_affiliation_strings":["Department of Clinical Neuroscience, Karolinska Institutet, Stockholm, Sweden"],"affiliations":[{"raw_affiliation_string":"Department of Clinical Neuroscience, Karolinska Institutet, Stockholm, Sweden","institution_ids":["https://openalex.org/I28166907"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056647153","display_name":"Jia-Wen Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210165726","display_name":"Shanghai Municipal Education Commission","ror":"https://ror.org/05tewj457","country_code":"CN","type":"government","lineage":["https://openalex.org/I4210165456","https://openalex.org/I4210165726"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jia-Wen Liu","raw_affiliation_strings":["Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726"]},{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726","https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040440605","display_name":"Bao\u2010Liang Lu","orcid":"https://orcid.org/0000-0001-8359-0058"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210165726","display_name":"Shanghai Municipal Education Commission","ror":"https://ror.org/05tewj457","country_code":"CN","type":"government","lineage":["https://openalex.org/I4210165456","https://openalex.org/I4210165726"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao-Liang Lu","raw_affiliation_strings":["Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726"]},{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726","https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056335002","display_name":"Wei\u2010Long Zheng","orcid":"https://orcid.org/0000-0002-9474-6369"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I4210165726","display_name":"Shanghai Municipal Education Commission","ror":"https://ror.org/05tewj457","country_code":"CN","type":"government","lineage":["https://openalex.org/I4210165456","https://openalex.org/I4210165726"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wei-Long Zheng","raw_affiliation_strings":["Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Center for Brain-Like Computing and Machine Intelligence, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726"]},{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, Department of Computer Science and Engineering, and the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I4210165726","https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100389973"],"corresponding_institution_ids":["https://openalex.org/I28166907"],"apc_list":null,"apc_paid":null,"fwci":1.682,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.84562782,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"16","issue":"3","first_page":"1543","last_page":"1556"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.958299994468689,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.958299994468689,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9559000134468079,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7264790534973145},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.708793580532074},{"id":"https://openalex.org/keywords/dual","display_name":"Dual (grammatical number)","score":0.6922816634178162},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.6551839113235474},{"id":"https://openalex.org/keywords/electroencephalography","display_name":"Electroencephalography","score":0.5808083415031433},{"id":"https://openalex.org/keywords/eye-movement","display_name":"Eye movement","score":0.5619335174560547},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.4825936555862427},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.47397011518478394},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4707266390323639},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.4399828016757965},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.42636919021606445},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3236303925514221},{"id":"https://openalex.org/keywords/neuroscience","display_name":"Neuroscience","score":0.08411598205566406}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7264790534973145},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.708793580532074},{"id":"https://openalex.org/C2780980858","wikidata":"https://www.wikidata.org/wiki/Q110022","display_name":"Dual (grammatical number)","level":2,"score":0.6922816634178162},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.6551839113235474},{"id":"https://openalex.org/C522805319","wikidata":"https://www.wikidata.org/wiki/Q179965","display_name":"Electroencephalography","level":2,"score":0.5808083415031433},{"id":"https://openalex.org/C153050134","wikidata":"https://www.wikidata.org/wiki/Q760256","display_name":"Eye movement","level":2,"score":0.5619335174560547},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.4825936555862427},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47397011518478394},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4707266390323639},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.4399828016757965},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.42636919021606445},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3236303925514221},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.08411598205566406},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taffc.2024.3524418","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2024.3524418","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.5600000023841858}],"awards":[{"id":"https://openalex.org/G5856684975","display_name":null,"funder_award_id":"62376158","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1947251450","https://openalex.org/W2001097956","https://openalex.org/W2023937851","https://openalex.org/W2031391348","https://openalex.org/W2038199522","https://openalex.org/W2056254268","https://openalex.org/W2084333685","https://openalex.org/W2084413241","https://openalex.org/W2085497785","https://openalex.org/W2090626380","https://openalex.org/W2109773980","https://openalex.org/W2132889650","https://openalex.org/W2140079265","https://openalex.org/W2142597810","https://openalex.org/W2143350951","https://openalex.org/W2247689161","https://openalex.org/W2516105285","https://openalex.org/W2525648609","https://openalex.org/W2619383789","https://openalex.org/W2776446922","https://openalex.org/W2786768213","https://openalex.org/W2945600262","https://openalex.org/W2962905870","https://openalex.org/W2979656641","https://openalex.org/W2982126608","https://openalex.org/W2997573100","https://openalex.org/W3000704572","https://openalex.org/W3003745987","https://openalex.org/W3009961050","https://openalex.org/W3014163061","https://openalex.org/W3080495370","https://openalex.org/W3094309768","https://openalex.org/W3123510091","https://openalex.org/W3159301005","https://openalex.org/W3177342940","https://openalex.org/W3207652202","https://openalex.org/W3215947932","https://openalex.org/W4206694574","https://openalex.org/W4221080645","https://openalex.org/W4225011203","https://openalex.org/W4294975119","https://openalex.org/W4310057041","https://openalex.org/W4313531231","https://openalex.org/W4313591463","https://openalex.org/W4385245566"],"related_works":["https://openalex.org/W2502115930","https://openalex.org/W2482350142","https://openalex.org/W4246396837","https://openalex.org/W3126451824","https://openalex.org/W1561927205","https://openalex.org/W3191453585","https://openalex.org/W4297672492","https://openalex.org/W4310988119","https://openalex.org/W3105646692","https://openalex.org/W4387914125"],"abstract_inverted_index":{"Emotion":[0],"recognition":[1,42],"is":[2,34,108,127],"a":[3,64,95,99,104],"fundamental":[4],"part":[5],"of":[6,32,39,86,178,188],"affective":[7],"computing,":[8],"obtaining":[9],"performance":[10,85,135,148],"gain":[11],"from":[12,76,94],"multimodal":[13,40,121],"methods.":[14],"Electroencephalography":[15],"(EEG)":[16],"and":[17,118,165],"eye":[18,47],"movements":[19,48],"are":[20,49],"extensively":[21],"used":[22],"as":[23],"they":[24],"contain":[25],"complementary":[26,74],"information.":[27],"However,":[28],"the":[29,36,73,83,112,120,130,146,154,171,181,186,194],"inconvenient":[30],"acquisition":[31],"EEG":[33],"hindering":[35],"extensive":[37],"adoption":[38],"emotion":[41,151,175],"in":[43,180],"daily":[44],"applications":[45],"while":[46],"more":[50],"convenient":[51],"to":[52,81,98,110,129,136,170],"collect":[53],"but":[54],"with":[55,68,193],"lower":[56],"performance.":[57],"To":[58],"tackle":[59],"this":[60],"issue,":[61],"we":[62],"propose":[63],"Constrained":[65],"Adversarial":[66],"Network":[67],"Dual":[69],"Attention":[70],"(CANDA),":[71],"exploiting":[72],"information":[75],"multiple":[77,137],"modalities":[78,117,190],"during":[79],"training":[80],"improve":[82],"test-time":[84],"single":[87,125],"easily":[88],"acquired":[89],"modality,":[90],"i.e.,":[91],"transferring":[92],"knowledge":[93],"stronger":[96],"modality":[97,126],"weaker":[100],"modality.":[101],"During":[102,123],"training,":[103],"common":[105,131],"joint":[106,182],"space":[107,132,183],"learned":[109],"diminish":[111],"distribution":[113,187],"discrepancy":[114],"among":[115],"different":[116,189],"incorporate":[119],"representations.":[122],"test,":[124],"converted":[128],"achieving":[133],"comparable":[134],"modalities.":[138],"Extensive":[139],"experiments":[140],"demonstrate":[141],"that":[142,185],"our":[143],"model":[144],"achieves":[145],"state-of-the-art":[147],"for":[149,174],"cross-modal":[150],"recognition.":[152,176],"Specifically,":[153],"mean":[155],"accuracy":[156],"increases":[157],"around":[158],"15%":[159,162],"on":[160,163,167],"SEED,":[161],"SEED-IV,":[164],"2%":[166],"SEED-V":[168],"compared":[169],"latest":[172],"baseline":[173],"Visualization":[177],"features":[179],"illustrates":[184],"aligns":[191],"together":[192],"discriminative":[195],"ability":[196],"regarding":[197],"various":[198],"emotions.":[199]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-09T08:11:56.329763","created_date":"2025-10-10T00:00:00"}