{"id":"https://openalex.org/W3193774712","doi":"https://doi.org/10.1109/taffc.2021.3106254","title":"Investigating Multisensory Integration in Emotion Recognition Through Bio-Inspired Computational Models","display_name":"Investigating Multisensory Integration in Emotion Recognition Through Bio-Inspired Computational Models","publication_year":2021,"publication_date":"2021-08-19","ids":{"openalex":"https://openalex.org/W3193774712","doi":"https://doi.org/10.1109/taffc.2021.3106254","mag":"3193774712"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2021.3106254","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2021.3106254","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research-repository.st-andrews.ac.uk/bitstream/10023/24022/1/Mansouri_Benssassi_2021_Investigating_multisensory_IEEETAC_ESMA_2020_AAM.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032008577","display_name":"Esma Mansouri-Benssassi","orcid":"https://orcid.org/0000-0003-3510-8129"},"institutions":[{"id":"https://openalex.org/I102298084","display_name":"Andrews University","ror":"https://ror.org/04aaa2n62","country_code":"US","type":"education","lineage":["https://openalex.org/I102298084"]},{"id":"https://openalex.org/I110813580","display_name":"St. Andrews University","ror":"https://ror.org/00nb5xj61","country_code":"US","type":"education","lineage":["https://openalex.org/I110813580"]},{"id":"https://openalex.org/I16835326","display_name":"University of St Andrews","ror":"https://ror.org/02wn5qz54","country_code":"GB","type":"education","lineage":["https://openalex.org/I16835326"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Esma Mansouri Benssassi","raw_affiliation_strings":["School of Computer Science, University of St Andrews, Andrews, U.K"],"raw_orcid":"https://orcid.org/0000-0003-3510-8129","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of St Andrews, Andrews, U.K","institution_ids":["https://openalex.org/I102298084","https://openalex.org/I110813580","https://openalex.org/I16835326"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100682481","display_name":"Juan Ye","orcid":"https://orcid.org/0000-0002-2838-6836"},"institutions":[{"id":"https://openalex.org/I102298084","display_name":"Andrews University","ror":"https://ror.org/04aaa2n62","country_code":"US","type":"education","lineage":["https://openalex.org/I102298084"]},{"id":"https://openalex.org/I110813580","display_name":"St. Andrews University","ror":"https://ror.org/00nb5xj61","country_code":"US","type":"education","lineage":["https://openalex.org/I110813580"]},{"id":"https://openalex.org/I16835326","display_name":"University of St Andrews","ror":"https://ror.org/02wn5qz54","country_code":"GB","type":"education","lineage":["https://openalex.org/I16835326"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Juan Ye","raw_affiliation_strings":["School of Computer Science, University of St Andrews, Andrews, U.K"],"raw_orcid":"https://orcid.org/0000-0002-2838-6836","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of St Andrews, Andrews, U.K","institution_ids":["https://openalex.org/I102298084","https://openalex.org/I110813580","https://openalex.org/I16835326"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2045,"currency":"USD","value_usd":2045},"apc_paid":null,"fwci":2.0538,"has_fulltext":true,"cited_by_count":20,"citation_normalized_percentile":{"value":0.86441814,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":"14","issue":"2","first_page":"906","last_page":"918"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12032","display_name":"Multisensory perception and integration","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10971","display_name":"Olfactory and Sensory Function Studies","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/2809","display_name":"Sensory Systems"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6933794617652893},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.6457142233848572},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6439085006713867},{"id":"https://openalex.org/keywords/multisensory-integration","display_name":"Multisensory integration","score":0.6079532504081726},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5267440676689148},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.49722054600715637},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4967041611671448},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.460105299949646},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.43317532539367676},{"id":"https://openalex.org/keywords/affective-computing","display_name":"Affective computing","score":0.42659685015678406},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1819269359111786},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.17569327354431152}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6933794617652893},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.6457142233848572},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6439085006713867},{"id":"https://openalex.org/C2780328332","wikidata":"https://www.wikidata.org/wiki/Q17166073","display_name":"Multisensory integration","level":3,"score":0.6079532504081726},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5267440676689148},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.49722054600715637},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4967041611671448},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.460105299949646},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.43317532539367676},{"id":"https://openalex.org/C6438553","wikidata":"https://www.wikidata.org/wiki/Q1185804","display_name":"Affective computing","level":2,"score":0.42659685015678406},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1819269359111786},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.17569327354431152},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/taffc.2021.3106254","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2021.3106254","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},{"id":"pmh:oai:research-repository.st-andrews.ac.uk:10023/24022","is_oa":true,"landing_page_url":"https://hdl.handle.net/10023/24022","pdf_url":"https://research-repository.st-andrews.ac.uk/bitstream/10023/24022/1/Mansouri_Benssassi_2021_Investigating_multisensory_IEEETAC_ESMA_2020_AAM.pdf","source":{"id":"https://openalex.org/S4306400230","display_name":"St Andrews Research Repository (St Andrews Research Repository)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I16835326","host_organization_name":"University of St Andrews","host_organization_lineage":["https://openalex.org/I16835326"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Journal article"}],"best_oa_location":{"id":"pmh:oai:research-repository.st-andrews.ac.uk:10023/24022","is_oa":true,"landing_page_url":"https://hdl.handle.net/10023/24022","pdf_url":"https://research-repository.st-andrews.ac.uk/bitstream/10023/24022/1/Mansouri_Benssassi_2021_Investigating_multisensory_IEEETAC_ESMA_2020_AAM.pdf","source":{"id":"https://openalex.org/S4306400230","display_name":"St Andrews Research Repository (St Andrews Research Repository)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I16835326","host_organization_name":"University of St Andrews","host_organization_lineage":["https://openalex.org/I16835326"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Journal article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.550000011920929,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3193774712.pdf","grobid_xml":"https://content.openalex.org/works/W3193774712.grobid-xml"},"referenced_works_count":89,"referenced_works":["https://openalex.org/W75502933","https://openalex.org/W97016928","https://openalex.org/W160386421","https://openalex.org/W178970520","https://openalex.org/W1522734439","https://openalex.org/W1534184781","https://openalex.org/W1570411240","https://openalex.org/W1971474447","https://openalex.org/W1972342515","https://openalex.org/W1976809315","https://openalex.org/W2006370340","https://openalex.org/W2026243162","https://openalex.org/W2029538510","https://openalex.org/W2045528981","https://openalex.org/W2053101950","https://openalex.org/W2058341538","https://openalex.org/W2065900404","https://openalex.org/W2073259742","https://openalex.org/W2079725295","https://openalex.org/W2091028497","https://openalex.org/W2100495367","https://openalex.org/W2135511277","https://openalex.org/W2144299089","https://openalex.org/W2149628368","https://openalex.org/W2191779130","https://openalex.org/W2259801182","https://openalex.org/W2281906198","https://openalex.org/W2283758531","https://openalex.org/W2333427390","https://openalex.org/W2395587375","https://openalex.org/W2406223855","https://openalex.org/W2406846344","https://openalex.org/W2407603194","https://openalex.org/W2409534643","https://openalex.org/W2464693423","https://openalex.org/W2476572832","https://openalex.org/W2506506742","https://openalex.org/W2508231302","https://openalex.org/W2511484725","https://openalex.org/W2525648609","https://openalex.org/W2546919788","https://openalex.org/W2555117749","https://openalex.org/W2583643061","https://openalex.org/W2584561145","https://openalex.org/W2588127243","https://openalex.org/W2608956662","https://openalex.org/W2612041314","https://openalex.org/W2624340939","https://openalex.org/W2703895418","https://openalex.org/W2766289529","https://openalex.org/W2774156856","https://openalex.org/W2786550078","https://openalex.org/W2789864788","https://openalex.org/W2803193013","https://openalex.org/W2805783416","https://openalex.org/W2806000748","https://openalex.org/W2807449773","https://openalex.org/W2809941642","https://openalex.org/W2884987480","https://openalex.org/W2886300652","https://openalex.org/W2889233055","https://openalex.org/W2889466822","https://openalex.org/W2894627692","https://openalex.org/W2897070044","https://openalex.org/W2898989925","https://openalex.org/W2901397387","https://openalex.org/W2901937441","https://openalex.org/W2911326064","https://openalex.org/W2950081742","https://openalex.org/W2954869531","https://openalex.org/W2964154335","https://openalex.org/W2977259558","https://openalex.org/W2977659243","https://openalex.org/W2981328134","https://openalex.org/W2997288337","https://openalex.org/W3036005995","https://openalex.org/W3104615707","https://openalex.org/W3104668471","https://openalex.org/W3125086424","https://openalex.org/W4299331580","https://openalex.org/W4312235822","https://openalex.org/W6603960082","https://openalex.org/W6631929317","https://openalex.org/W6651155528","https://openalex.org/W6685350579","https://openalex.org/W6726873649","https://openalex.org/W6752162782","https://openalex.org/W6752993679","https://openalex.org/W6764972056"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W2167701463","https://openalex.org/W2110287964","https://openalex.org/W4307407935","https://openalex.org/W649759291"],"abstract_inverted_index":{"Emotion":[0],"understanding":[1,20],"represents":[2],"a":[3,49],"core":[4],"aspect":[5],"of":[6,31,60,101],"human":[7],"communication.":[8],"Our":[9],"social":[10,27],"behaviours":[11],"are":[12,58,123,137],"closely":[13],"linked":[14],"to":[15,146,171,184,202],"expressing":[16],"our":[17],"emotions":[18],"and":[19,23,43,65,86,115,125,144,155,200],"others\u2019":[21],"emotional":[22],"mental":[24],"states":[25],"through":[26,117],"signals.":[28],"The":[29,120,135],"majority":[30],"the":[32,62,88,105,163,172,191],"existing":[33],"work":[34],"proceeds":[35],"by":[36,110],"extracting":[37],"meaningful":[38],"features":[39],"from":[40],"each":[41],"modality":[42,82],"applying":[44],"fusion":[45,149],"techniques":[46,57],"either":[47],"at":[48],"feature":[50],"level":[51],"or":[52],"decision":[53],"level.":[54],"However,":[55],"these":[56],"incapable":[59],"translating":[61],"constant":[63,71,186],"talk":[64,72,187],"feedback":[66],"between":[67,188],"different":[68,99],"modalities.":[69],"Such":[70],"is":[73],"particularly":[74],"important":[75],"in":[76,104,198],"continuous":[77],"emotion":[78],"recognition,":[79],"where":[80],"one":[81],"can":[83],"predict,":[84],"enhance":[85],"complement":[87],"other.":[89],"This":[90],"article":[91,180],"proposes":[92],"three":[93,164],"multisensory":[94,102],"integration":[95,103,109,116],"models,":[96],"based":[97],"on":[98],"pathways":[100],"brain;":[106],"that":[107,162],"is,":[108],"convergence,":[111],"early":[112],"cross-modal":[113],"enhancement,":[114],"neural":[118,129],"synchrony.":[119],"proposed":[121,165],"models":[122,136,166],"designed":[124],"implemented":[126],"using":[127,139],"third-generation":[128],"networks,":[130],"Spiking":[131],"Neural":[132],"Networks":[133],"(SNN).":[134],"evaluated":[138],"widely":[140],"adopted,":[141],"third-party":[142],"datasets":[143],"compared":[145],"state-of-the-art":[147,173],"multimodal":[148],"techniques,":[150],"such":[151],"as":[152],"early,":[153],"late":[154],"deep":[156],"learning":[157,175],"fusion.":[158],"Evaluation":[159],"results":[160,170],"show":[161],"have":[167],"achieved":[168],"comparable":[169],"supervised":[174],"techniques.":[176],"More":[177],"importantly,":[178],"this":[179],"demonstrates":[181],"plausible":[182],"ways":[183],"translate":[185],"modalities":[189],"during":[190],"training":[192],"phase,":[193],"which":[194],"also":[195],"brings":[196],"advantages":[197],"generalisation":[199],"robustness":[201],"noise.":[203]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":2}],"updated_date":"2026-08-28T12:50:07.497085","created_date":"2021-08-30T00:00:00"}