{"id":"https://openalex.org/W4401537007","doi":"https://doi.org/10.1109/tcds.2024.3442203","title":"Multimodal Emotion Fusion Mechanism and Empathetic Responses in Companion Robots","display_name":"Multimodal Emotion Fusion Mechanism and Empathetic Responses in Companion Robots","publication_year":2024,"publication_date":"2024-08-13","ids":{"openalex":"https://openalex.org/W4401537007","doi":"https://doi.org/10.1109/tcds.2024.3442203"},"language":"en","primary_location":{"id":"doi:10.1109/tcds.2024.3442203","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2024.3442203","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100359300","display_name":"Xiaofeng Liu","orcid":"https://orcid.org/0000-0003-1310-6739"},"institutions":[{"id":"https://openalex.org/I163340411","display_name":"Hohai University","ror":"https://ror.org/01wd4xt90","country_code":"CN","type":"education","lineage":["https://openalex.org/I163340411"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xiaofeng Liu","raw_affiliation_strings":["College of Artificial Intelligence and Automation, Hohai University, Changzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-1310-6739","affiliations":[{"raw_affiliation_string":"College of Artificial Intelligence and Automation, Hohai University, Changzhou, China","institution_ids":["https://openalex.org/I163340411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006609598","display_name":"Qincheng Lv","orcid":"https://orcid.org/0009-0004-5557-5706"},"institutions":[{"id":"https://openalex.org/I163340411","display_name":"Hohai University","ror":"https://ror.org/01wd4xt90","country_code":"CN","type":"education","lineage":["https://openalex.org/I163340411"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qincheng Lv","raw_affiliation_strings":["College of Information Science and Engineering, Hohai University, Changzhou, China"],"raw_orcid":"https://orcid.org/0009-0004-5557-5706","affiliations":[{"raw_affiliation_string":"College of Information Science and Engineering, Hohai University, Changzhou, China","institution_ids":["https://openalex.org/I163340411"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080670840","display_name":"Jie Li","orcid":"https://orcid.org/0000-0002-2977-8559"},"institutions":[{"id":"https://openalex.org/I83519826","display_name":"Nanjing Medical University","ror":"https://ror.org/059gcgy73","country_code":"CN","type":"education","lineage":["https://openalex.org/I83519826"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Li","raw_affiliation_strings":["Engineering Research Center of Intelligent Theranostics Technology and Instruments, Ministry of Education, and the School of Biomedical Engineering and Informatics, Nanjing Medical University, Nanjing, China","Engineering Research Center of Intelligent Theranostics Technology and Instruments, Ministry of Education, School of Biomedical Engineering and Informatics, Nanjing Medical University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-2977-8559","affiliations":[{"raw_affiliation_string":"Engineering Research Center of Intelligent Theranostics Technology and Instruments, Ministry of Education, and the School of Biomedical Engineering and Informatics, Nanjing Medical University, Nanjing, China","institution_ids":["https://openalex.org/I83519826"]},{"raw_affiliation_string":"Engineering Research Center of Intelligent Theranostics Technology and Instruments, Ministry of Education, School of Biomedical Engineering and Informatics, Nanjing Medical University, Nanjing, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053061988","display_name":"Siyang Song","orcid":"https://orcid.org/0000-0003-2339-5685"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210116723","display_name":"Robotics Research (United States)","ror":"https://ror.org/020w2fr77","country_code":"US","type":"company","lineage":["https://openalex.org/I4210116723"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Siyang Song","raw_affiliation_strings":["Affective Intelligence and Robotics Lab, Department of Computer Science and Technology, University of Cambridge, Cambridge, U.K","AFAR Lab, Department of Computer Science and Technology, University of Cambridge, Cambridge, United Kingdom"],"raw_orcid":"https://orcid.org/0000-0003-2339-5685","affiliations":[{"raw_affiliation_string":"Affective Intelligence and Robotics Lab, Department of Computer Science and Technology, University of Cambridge, Cambridge, U.K","institution_ids":["https://openalex.org/I4210116723"]},{"raw_affiliation_string":"AFAR Lab, Department of Computer Science and Technology, University of Cambridge, Cambridge, United Kingdom","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5091768977","display_name":"Angelo Cangelosi","orcid":"https://orcid.org/0000-0002-4709-2243"},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Angelo Cangelosi","raw_affiliation_strings":["Machine Learning and Optimisation, University of Manchester, Manchester, U.K"],"raw_orcid":"https://orcid.org/0000-0002-4709-2243","affiliations":[{"raw_affiliation_string":"Machine Learning and Optimisation, University of Manchester, Manchester, U.K","institution_ids":["https://openalex.org/I28407311"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5100359300"],"corresponding_institution_ids":["https://openalex.org/I163340411"],"apc_list":null,"apc_paid":null,"fwci":3.0544,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.91077454,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"17","issue":"2","first_page":"271","last_page":"286"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.745337963104248},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.7090999484062195},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5729509592056274},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.5090855956077576},{"id":"https://openalex.org/keywords/fusion-mechanism","display_name":"Fusion mechanism","score":0.4617611765861511},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.42311254143714905},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.39990663528442383},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.321347713470459},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3154710829257965},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.206749826669693},{"id":"https://openalex.org/keywords/lipid-bilayer-fusion","display_name":"Lipid bilayer fusion","score":0.08918550610542297}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.745337963104248},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.7090999484062195},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5729509592056274},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.5090855956077576},{"id":"https://openalex.org/C173414695","wikidata":"https://www.wikidata.org/wiki/Q5510276","display_name":"Fusion mechanism","level":4,"score":0.4617611765861511},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.42311254143714905},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.39990663528442383},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.321347713470459},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3154710829257965},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.206749826669693},{"id":"https://openalex.org/C103038307","wikidata":"https://www.wikidata.org/wiki/Q6556360","display_name":"Lipid bilayer fusion","level":3,"score":0.08918550610542297},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tcds.2024.3442203","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2024.3442203","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/d833b6b9-9ba9-4717-b5ef-9700416282bf","is_oa":false,"landing_page_url":"https://research.manchester.ac.uk/en/publications/d833b6b9-9ba9-4717-b5ef-9700416282bf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Liu, X, Lv, Q, Li, J, Song, S & Cangelosi, A 2025, 'Multimodal Emotion Fusion Mechanism and Empathetic Responses in Companion Robots', IEEE Transactions on Cognitive and Developmental Systems. https://doi.org/10.1109/TCDS.2024.3442203","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2975661462","display_name":null,"funder_award_id":"62276090","funder_id":"https://openalex.org/F4320329293","funder_display_name":"Data Center of Management Science, National Natural Science Foundation of China - Peking University"},{"id":"https://openalex.org/G330974250","display_name":null,"funder_award_id":"2018AAA0100803","funder_id":"https://openalex.org/F4320338173","funder_display_name":"Chinese Polar Environment Comprehensive Investigation and Assessment Programmes"},{"id":"https://openalex.org/G5121403982","display_name":null,"funder_award_id":"62203150","funder_id":"https://openalex.org/F4320329293","funder_display_name":"Data Center of Management Science, National Natural Science Foundation of China - Peking University"}],"funders":[{"id":"https://openalex.org/F4320329293","display_name":"Data Center of Management Science, National Natural Science Foundation of China - Peking University","ror":null},{"id":"https://openalex.org/F4320338173","display_name":"Chinese Polar Environment Comprehensive Investigation and Assessment Programmes","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":77,"referenced_works":["https://openalex.org/W2017561954","https://openalex.org/W2103943262","https://openalex.org/W2106115875","https://openalex.org/W2135826624","https://openalex.org/W2149628368","https://openalex.org/W2161576359","https://openalex.org/W2162671057","https://openalex.org/W2164598857","https://openalex.org/W2556418146","https://openalex.org/W2581686065","https://openalex.org/W2606867274","https://openalex.org/W2740440254","https://openalex.org/W2774054212","https://openalex.org/W2786768213","https://openalex.org/W2787581402","https://openalex.org/W2789598332","https://openalex.org/W2791017690","https://openalex.org/W2791452829","https://openalex.org/W2798066193","https://openalex.org/W2894248579","https://openalex.org/W2899785433","https://openalex.org/W2905628412","https://openalex.org/W2908671501","https://openalex.org/W2910166529","https://openalex.org/W2924126491","https://openalex.org/W2940314039","https://openalex.org/W2952478944","https://openalex.org/W2962931510","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964346351","https://openalex.org/W2967182021","https://openalex.org/W2972273766","https://openalex.org/W3000204199","https://openalex.org/W3003908700","https://openalex.org/W3006715241","https://openalex.org/W3022569189","https://openalex.org/W3025630664","https://openalex.org/W3034849760","https://openalex.org/W3036120140","https://openalex.org/W3037572520","https://openalex.org/W3042803450","https://openalex.org/W3093051361","https://openalex.org/W3102476796","https://openalex.org/W3111911126","https://openalex.org/W3114195756","https://openalex.org/W3116201721","https://openalex.org/W3121492439","https://openalex.org/W3124815229","https://openalex.org/W3128412859","https://openalex.org/W3128486603","https://openalex.org/W3137809055","https://openalex.org/W3144848010","https://openalex.org/W3186105911","https://openalex.org/W3206529771","https://openalex.org/W3215440557","https://openalex.org/W4206973639","https://openalex.org/W4211156140","https://openalex.org/W4281259984","https://openalex.org/W4288804239","https://openalex.org/W4309166667","https://openalex.org/W4312345918","https://openalex.org/W4321439011","https://openalex.org/W4361030040","https://openalex.org/W4364302344","https://openalex.org/W4367310920","https://openalex.org/W4367353919","https://openalex.org/W4380609838","https://openalex.org/W4385478147","https://openalex.org/W4386025684","https://openalex.org/W4387968034","https://openalex.org/W4387968147","https://openalex.org/W4387969719","https://openalex.org/W4388188050","https://openalex.org/W4404480514","https://openalex.org/W6850222400","https://openalex.org/W6853235891"],"related_works":["https://openalex.org/W2988498288","https://openalex.org/W1991764124","https://openalex.org/W2786308501","https://openalex.org/W2012247345","https://openalex.org/W4318719019","https://openalex.org/W4385065518","https://openalex.org/W2145243726","https://openalex.org/W2351649588","https://openalex.org/W4378188910","https://openalex.org/W2437857815"],"abstract_inverted_index":{"The":[0],"ability":[1],"of":[2,45,56,67,71,175],"humanoid":[3,58,64],"robots":[4,95],"to":[5,80,87,97,133,153,198],"exhibit":[6],"empathetic":[7,135,155,194],"facial":[8,30,49,73,84],"expressions":[9,50,156],"and":[10,75,118,143,185,190],"provide":[11],"corresponding":[12],"responses":[13,136],"is":[14],"essential":[15],"for":[16],"natural":[17,83],"human\u2013robot":[18],"interaction.":[19],"To":[20],"enhance":[21],"this,":[22],"we":[23,41,61,102,128],"integrate":[24,129],"the":[25,43,53,69,89,122,130,151,173,176],"GPT3.5":[26],"model":[27,132],"with":[28],"a":[29,35,57,63,104,160],"expression":[31],"recognition":[32,38,112,125,181],"model,":[33],"creating":[34],"multimodal":[36,110,179],"emotion":[37,111,124,180],"system.":[39],"Additionally,":[40],"address":[42],"challenge":[44],"realistically":[46],"mimicking":[47],"human":[48],"by":[51],"designing":[52],"physical":[54],"structure":[55],"robot.":[59],"Initially,":[60],"develop":[62],"robot":[65,152],"capable":[66],"adjusting":[68],"positions":[70],"its":[72],"organs":[74],"neck":[76],"through":[77],"servo":[78],"displacement":[79],"achieve":[81,159],"more":[82,161],"expressions.":[85],"Subsequently,":[86],"overcome":[88],"current":[90],"limitation":[91],"where":[92],"emotional":[93,141],"interaction":[94,164],"struggle":[96],"accurately":[98],"recognize":[99],"user":[100,140],"emotions,":[101],"introduce":[103],"coupled":[105,177],"generative":[106],"pretrained":[107],"transformer":[108],"(GPT)-based":[109],"method":[113,182],"that":[114,157,172],"utilizes":[115],"both":[116],"text":[117,184],"images,":[119],"thereby":[120],"enhancing":[121],"robot's":[123],"accuracy.":[126],"Finally,":[127],"GPT-3.5":[131],"generate":[134],"based":[137],"on":[138,168],"recognized":[139],"states":[142],"language":[144],"text,":[145],"which":[146],"are":[147],"then":[148],"mapped":[149],"onto":[150],"enable":[154],"can":[158],"comfortable":[162],"human\u2013machine":[163],"experience.":[165],"Experimental":[166],"results":[167],"benchmark":[169],"databases":[170],"demonstrate":[171],"performance":[174],"GPT-based":[178],"using":[183],"images":[186],"outperforms":[187],"other":[188],"approaches,":[189],"it":[191],"possesses":[192],"unique":[193],"response":[195],"capabilities":[196],"relative":[197],"alternative":[199],"methods.":[200]},"counts_by_year":[{"year":2025,"cited_by_count":5}],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-10-10T00:00:00"}