{"id":"https://openalex.org/W4396877672","doi":"https://doi.org/10.1109/tim.2024.3400341","title":"MGMS: A Modality-General and Modality-Specific Learning Model Using EEG and Facial Expression for Hearing-Impaired Emotion Recognition","display_name":"MGMS: A Modality-General and Modality-Specific Learning Model Using EEG and Facial Expression for Hearing-Impaired Emotion Recognition","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4396877672","doi":"https://doi.org/10.1109/tim.2024.3400341"},"language":"en","primary_location":{"id":"doi:10.1109/tim.2024.3400341","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tim.2024.3400341","pdf_url":null,"source":{"id":"https://openalex.org/S10892749","display_name":"IEEE Transactions on Instrumentation and Measurement","issn_l":"0018-9456","issn":["0018-9456","1557-9662"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Instrumentation and Measurement","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101555680","display_name":"Qingzhou Wu","orcid":"https://orcid.org/0000-0003-4997-7966"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingzhou Wu","raw_affiliation_strings":["School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-4997-7966","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101923146","display_name":"Mu Zhu","orcid":"https://orcid.org/0000-0003-3224-5654"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mu Zhu","raw_affiliation_strings":["School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-3224-5654","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098008977","display_name":"Wenhui Xu","orcid":"https://orcid.org/0009-0009-6622-211X"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenhui Xu","raw_affiliation_strings":["School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0009-0009-6622-211X","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071117560","display_name":"Junchi Wang","orcid":"https://orcid.org/0000-0001-9081-702X"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junchi Wang","raw_affiliation_strings":["School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0001-9081-702X","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059381970","display_name":"Zemin Mao","orcid":"https://orcid.org/0000-0003-4698-3340"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zemin Mao","raw_affiliation_strings":["Technical College for the Deaf, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0003-4698-3340","affiliations":[{"raw_affiliation_string":"Technical College for the Deaf, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015388584","display_name":"Qiang Gao","orcid":"https://orcid.org/0000-0001-9357-4967"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qiang Gao","raw_affiliation_strings":["Tianjin Key Laboratory for Control Theory and Applications in Complicated Systems, TUT Maritime College, Tianjin University of Technology, Tianjin, China","TUT Maritime College, Tianjin Key Laboratory for Control Theory and Applications in Complicated Systems, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0001-9357-4967","affiliations":[{"raw_affiliation_string":"Tianjin Key Laboratory for Control Theory and Applications in Complicated Systems, TUT Maritime College, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]},{"raw_affiliation_string":"TUT Maritime College, Tianjin Key Laboratory for Control Theory and Applications in Complicated Systems, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5027879510","display_name":"Yu Song","orcid":"https://orcid.org/0000-0002-9295-7795"},"institutions":[{"id":"https://openalex.org/I136765683","display_name":"Tianjin University of Technology","ror":"https://ror.org/00zbe0w13","country_code":"CN","type":"education","lineage":["https://openalex.org/I136765683"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Song","raw_affiliation_strings":["School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China"],"raw_orcid":"https://orcid.org/0000-0002-9295-7795","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Automation, Tianjin University of Technology, Tianjin, China","institution_ids":["https://openalex.org/I136765683"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I136765683"],"apc_list":null,"apc_paid":null,"fwci":1.6893,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.8412136,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"73","issue":null,"first_page":"1","last_page":"11"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.7878999710083008,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10860","display_name":"Speech and Audio Processing","score":0.7878999710083008,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.7717000246047974,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.6869999766349792,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.83697909116745},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.7010946869850159},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.6681538820266724},{"id":"https://openalex.org/keywords/electroencephalography","display_name":"Electroencephalography","score":0.6343517899513245},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5519791841506958},{"id":"https://openalex.org/keywords/hearing-impaired","display_name":"Hearing impaired","score":0.45154377818107605},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42133185267448425},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.33908432722091675},{"id":"https://openalex.org/keywords/audiology","display_name":"Audiology","score":0.32328182458877563},{"id":"https://openalex.org/keywords/neuroscience","display_name":"Neuroscience","score":0.23218229413032532},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.10445842146873474}],"concepts":[{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.83697909116745},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.7010946869850159},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.6681538820266724},{"id":"https://openalex.org/C522805319","wikidata":"https://www.wikidata.org/wiki/Q179965","display_name":"Electroencephalography","level":2,"score":0.6343517899513245},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5519791841506958},{"id":"https://openalex.org/C3020736805","wikidata":"https://www.wikidata.org/wiki/Q16035842","display_name":"Hearing impaired","level":2,"score":0.45154377818107605},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42133185267448425},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33908432722091675},{"id":"https://openalex.org/C548259974","wikidata":"https://www.wikidata.org/wiki/Q569965","display_name":"Audiology","level":1,"score":0.32328182458877563},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.23218229413032532},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.10445842146873474}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tim.2024.3400341","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tim.2024.3400341","pdf_url":null,"source":{"id":"https://openalex.org/S10892749","display_name":"IEEE Transactions on Instrumentation and Measurement","issn_l":"0018-9456","issn":["0018-9456","1557-9662"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Instrumentation and Measurement","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7099999785423279,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G2512941143","display_name":null,"funder_award_id":"62103299","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W236742792","https://openalex.org/W1947251450","https://openalex.org/W2043181832","https://openalex.org/W2115252128","https://openalex.org/W2149628368","https://openalex.org/W2783276282","https://openalex.org/W2972055227","https://openalex.org/W2982299617","https://openalex.org/W2998830321","https://openalex.org/W3009332477","https://openalex.org/W3043900038","https://openalex.org/W3084140714","https://openalex.org/W3108087271","https://openalex.org/W3120929420","https://openalex.org/W3158362529","https://openalex.org/W3159301005","https://openalex.org/W3159759379","https://openalex.org/W3163470448","https://openalex.org/W3176404283","https://openalex.org/W3186105911","https://openalex.org/W3207494443","https://openalex.org/W3208476368","https://openalex.org/W3215334947","https://openalex.org/W4205439218","https://openalex.org/W4206694574","https://openalex.org/W4210683997","https://openalex.org/W4211249450","https://openalex.org/W4213304810","https://openalex.org/W4214729239","https://openalex.org/W4220799835","https://openalex.org/W4221162872","https://openalex.org/W4226213590","https://openalex.org/W4229374389","https://openalex.org/W4286640379","https://openalex.org/W4292338326","https://openalex.org/W4293261951","https://openalex.org/W4306250392","https://openalex.org/W4310592224","https://openalex.org/W4311174819","https://openalex.org/W4319663643","https://openalex.org/W4319999789","https://openalex.org/W4361987556","https://openalex.org/W4363649593","https://openalex.org/W4366983762","https://openalex.org/W4385245566","https://openalex.org/W6677618333","https://openalex.org/W6734335776"],"related_works":["https://openalex.org/W2385859805","https://openalex.org/W2922348724","https://openalex.org/W2530972254","https://openalex.org/W200322357","https://openalex.org/W2130428257","https://openalex.org/W4308951944","https://openalex.org/W2057366091","https://openalex.org/W2049513647","https://openalex.org/W2988848585","https://openalex.org/W4233722919"],"abstract_inverted_index":{"Currently,":[0],"most":[1],"research":[2],"on":[3,8],"emotion":[4,18],"recognition":[5],"primarily":[6],"relies":[7],"single-modal":[9],"physiological":[10],"or":[11],"non-physiological":[12],"methods,":[13],"overlooking":[14],"the":[15,34,37,48,59,73,82,100,128,142,150,161,173,187],"complementarity":[16],"of":[17,36,61,75,114,181,190],"representation":[19],"across":[20],"different":[21,95],"modalities.":[22],"Individuals":[23],"with":[24,42],"hearing":[25],"impairments":[26],"may":[27],"experience":[28],"emotional":[29,38],"cognitive":[30],"biases":[31],"due":[32],"to":[33,57,159],"loss":[35,167],"acquisition":[39],"pathway":[40],"associated":[41],"hearing.":[43],"Therefore,":[44],"this":[45],"study":[46],"introduces":[47],"modality-general":[49,143],"and":[50,70,78,98,119,131,144,194],"modality-specific":[51,145],"(MGMS)":[52],"learning":[53],"model,":[54],"which":[55,124],"aims":[56],"examine":[58],"emotions":[60],"hearing-impaired":[62],"individuals":[63],"in":[64],"four":[65,162],"categories":[66],"(fear,":[67],"happy,":[68],"neutral,":[69],"sad)":[71],"through":[72],"fusion":[74],"electroencephalogram":[76],"(EEG)":[77],"facial":[79,115,133,197],"expression.":[80],"Specifically,":[81],"differential":[83],"entropy":[84],"(DE)":[85],"features":[86,118,121,146,163],"are":[87,122,125,147],"manually":[88],"extracted":[89,126],"from":[90,149],"each":[91],"EEG":[92,193],"channel":[93],"by":[94,105,127],"brain":[96],"regions,":[97],"then":[99],"spatial":[101],"information":[102],"is":[103,157],"captured":[104],"a":[106,139,154,165],"Long":[107],"Short-Term":[108],"Memory":[109],"(LSTM)":[110],"network.":[111],"In":[112],"terms":[113],"expression,":[116],"texture":[117],"geometric":[120],"combined":[123],"ResNet":[129],"network":[130],"68":[132],"key":[134],"points,":[135],"respectively.":[136],"By":[137],"constructing":[138],"general-specific":[140],"discriminator,":[141],"separated":[148],"two":[151],"modes.":[152],"Furthermore,":[153],"Transformer":[155],"encoder":[156],"employed":[158],"classify":[160],"using":[164],"cross-entropy":[166],"function.":[168],"Experimental":[169],"results":[170],"demonstrate":[171],"that":[172],"proposed":[174],"methods":[175],"achieve":[176],"an":[177],"average":[178],"classification":[179],"accuracy":[180],"86.01%":[182],"for":[183,192,196],"subject-dependent":[184],"classification,":[185],"surpassing":[186],"respective":[188],"accuracies":[189],"65.12%":[191],"59.86%":[195],"expressions.":[198]},"counts_by_year":[{"year":2025,"cited_by_count":6}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}