{"id":"https://openalex.org/W2979656641","doi":"https://doi.org/10.1109/embc.2019.8856563","title":"Multimodal Emotion Recognition from Eye Image, Eye Movement and EEG Using Deep Neural Networks","display_name":"Multimodal Emotion Recognition from Eye Image, Eye Movement and EEG Using Deep Neural Networks","publication_year":2019,"publication_date":"2019-07-01","ids":{"openalex":"https://openalex.org/W2979656641","doi":"https://doi.org/10.1109/embc.2019.8856563","mag":"2979656641","pmid":"https://pubmed.ncbi.nlm.nih.gov/31946536"},"language":"en","primary_location":{"id":"doi:10.1109/embc.2019.8856563","doi":"https://doi.org/10.1109/embc.2019.8856563","is_oa":false,"landing_page_url":"https://doi.org/10.1109/embc.2019.8856563","pdf_url":null,"source":{"id":"https://openalex.org/S4363608786","display_name":"Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)","issn_l":"2375-7477","issn":["2375-7477"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["doyens","jufo-1","ki-jl-1","medline","russia-white-list-3"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 41st Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071102667","display_name":"Jiang-Jian Guo","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang-Jian Guo","raw_affiliation_strings":["Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058594445","display_name":"Rong Feng Zhou","orcid":"https://orcid.org/0000-0003-3542-7453","observed_orcids":["https://orcid.org/0000-0003-3542-7453"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rong Zhou","raw_affiliation_strings":["Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045300183","display_name":"Li-Ming Zhao","orcid":"https://orcid.org/0000-0002-1467-1230","observed_orcids":["https://orcid.org/0000-0002-1467-1230"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li-Ming Zhao","raw_affiliation_strings":["Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040440605","display_name":"Bao\u2010Liang Lu","orcid":"https://orcid.org/0000-0001-8359-0058","observed_orcids":["https://orcid.org/0000-0001-8359-0058"]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bao-Liang Lu","raw_affiliation_strings":["Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Brain-Like Computing and Machine Intelligence, the Key Laboratory of Shanghai Education Commission for Intelligent Interaction and Cognitive Engineering and the Brain Science and Technology Research Center, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I183067930"],"apc_list":null,"apc_paid":null,"fwci":8.8333,"has_fulltext":false,"cited_by_count":73,"citation_normalized_percentile":{"value":0.97978324,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":"2019","issue":null,"first_page":"3071","last_page":"3074"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-emotion-recognition","display_name":"multimodal emotion recognition","score":0.9919999837875366},{"id":"https://openalex.org/keywords/eye-tracking-glasses","display_name":"eye-tracking glasses","score":0.718999981880188},{"id":"https://openalex.org/keywords/eye-images","display_name":"eye images","score":0.9150000214576721},{"id":"https://openalex.org/keywords/eye-movements","display_name":"eye movements","score":0.8989999890327454},{"id":"https://openalex.org/keywords/electroencephalography","display_name":"EEG","score":0.9409999847412109},{"id":"https://openalex.org/keywords/early-fusion","display_name":"feature-level fusion","score":0.5899999737739563},{"id":"https://openalex.org/keywords/emotion-classification","display_name":"emotion classification","score":0.5540000200271606},{"id":"https://openalex.org/keywords/data-fusion","display_name":"data fusion","score":0.40400001406669617}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7636154890060425},{"id":"https://openalex.org/C153050134","wikidata":"https://www.wikidata.org/wiki/Q760256","display_name":"Eye movement","level":2,"score":0.7165948748588562},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7119463086128235},{"id":"https://openalex.org/C522805319","wikidata":"https://www.wikidata.org/wiki/Q179965","display_name":"Electroencephalography","level":2,"score":0.6665716171264648},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.5967955589294434},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.586858868598938},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.5558760166168213},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5485332012176514},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5193842053413391},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.4715438187122345},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.44941458106040955},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.4176177382469177},{"id":"https://openalex.org/C69744172","wikidata":"https://www.wikidata.org/wiki/Q860822","display_name":"Image fusion","level":3,"score":0.4167560935020447},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.38931459188461304},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.3102707266807556},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11765971779823303},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C118552586","wikidata":"https://www.wikidata.org/wiki/Q7867","display_name":"Psychiatry","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D004569","descriptor_name":"Electroencephalography","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D004644","descriptor_name":"Emotions","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D005133","descriptor_name":"Eye Movements","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":2,"locations":[{"id":"doi:10.1109/embc.2019.8856563","doi":"https://doi.org/10.1109/embc.2019.8856563","is_oa":false,"landing_page_url":"https://doi.org/10.1109/embc.2019.8856563","pdf_url":null,"source":{"id":"https://openalex.org/S4363608786","display_name":"Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)","issn_l":"2375-7477","issn":["2375-7477"],"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":["doyens","jufo-1","ki-jl-1","medline","russia-white-list-3"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 41st Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC)","raw_type":"proceedings-article"},{"id":"pmid:31946536","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/31946536","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Annual International Conference of the IEEE Engineering in Medicine and Biology Society. IEEE Engineering in Medicine and Biology Society. Annual International Conference","raw_type":"Journal Article"}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W1821352908","https://openalex.org/W1947251450","https://openalex.org/W1955904464","https://openalex.org/W1970727126","https://openalex.org/W1974210421","https://openalex.org/W1990306865","https://openalex.org/W2079735306","https://openalex.org/W2080080403","https://openalex.org/W2085497785","https://openalex.org/W2109773980","https://openalex.org/W2163605009","https://openalex.org/W2194775991","https://openalex.org/W2407512993","https://openalex.org/W2525648609","https://openalex.org/W2737784625","https://openalex.org/W2945331847","https://openalex.org/W2945600262"],"related_works":["https://openalex.org/W2085497785","https://openalex.org/W2945600262","https://openalex.org/W2786768213","https://openalex.org/W2407512993","https://openalex.org/W4385519223","https://openalex.org/W4403023468","https://openalex.org/W4387092317","https://openalex.org/W3186724373","https://openalex.org/W4361987556","https://openalex.org/W4412972642"],"abstract_inverted_index":{"In":[0,19],"consideration":[1],"of":[2,5,16,25,53,58,63,87,132],"the":[3,23,54,76,83,93,99,111,116],"complexity":[4],"recording":[6],"electroencephalography(EEG),":[7],"some":[8],"researchers":[9],"are":[10],"trying":[11],"to":[12,21,39,75],"find":[13,97],"new":[14],"features":[15,79,125],"emotion":[17,31],"recognition.":[18],"order":[20],"investigate":[22],"potential":[24],"eye":[26,37,45,119,122],"tracking":[27],"glasses":[28],"for":[29,106],"multimodal":[30],"recognition,":[32],"we":[33,96],"collect":[34],"and":[35,47,60,69,121],"use":[36],"images":[38],"classify":[40],"five":[41,108],"emotions":[42],"along":[43],"with":[44,118],"movements":[46],"EEG.":[48],"We":[49],"compare":[50],"four":[51],"combinations":[52],"three":[55,100],"different":[56],"types":[57],"data":[59],"two":[61],"kinds":[62],"fusion":[64,68,78,124],"methods,":[65],"feature":[66],"level":[67],"Bimodal":[70],"Deep":[71],"AutoEncoder":[72],"(BDAE).":[73],"According":[74],"three-modality":[77],"generated":[80],"by":[81],"BDAE,":[82],"best":[84],"mean":[85],"accuracy":[86,131],"79.63%":[88],"is":[89],"achieved.":[90],"By":[91],"analyzing":[92],"confusion":[94],"matrices,":[95],"that":[98,115],"modalities":[101],"can":[102,126],"provide":[103],"complementary":[104],"information":[105],"recognizing":[107],"emotions.":[109],"Meanwhile,":[110],"experimental":[112],"results":[113],"indicate":[114],"classifiers":[117],"image":[120],"movement":[123],"achieve":[127],"a":[128],"comparable":[129],"classification":[130],"71.99%.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":13},{"year":2021,"cited_by_count":13},{"year":2020,"cited_by_count":2}],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2019-10-18T00:00:00"}