{"id":"https://openalex.org/W4404035171","doi":"https://doi.org/10.1109/tcsvt.2024.3491098","title":"DEP-Former: Multimodal Depression Recognition Based on Facial Expressions and Audio Features via Emotional Changes","display_name":"DEP-Former: Multimodal Depression Recognition Based on Facial Expressions and Audio Features via Emotional Changes","publication_year":2024,"publication_date":"2024-11-04","ids":{"openalex":"https://openalex.org/W4404035171","doi":"https://doi.org/10.1109/tcsvt.2024.3491098"},"language":"en","primary_location":{"id":"doi:10.1109/tcsvt.2024.3491098","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3491098","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101621183","display_name":"Jiayu Ye","orcid":"https://orcid.org/0000-0003-0368-9651"},"institutions":[{"id":"https://openalex.org/I139024713","display_name":"Guangdong University of Technology","ror":"https://ror.org/04azbjn80","country_code":"CN","type":"education","lineage":["https://openalex.org/I139024713"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiayu Ye","raw_affiliation_strings":["School of Computer Science, Guangdong University of Technology, Guangzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-0368-9651","affiliations":[{"raw_affiliation_string":"School of Computer Science, Guangdong University of Technology, Guangzhou, China","institution_ids":["https://openalex.org/I139024713"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102912826","display_name":"Yanhong Yu","orcid":"https://orcid.org/0000-0002-6547-6320"},"institutions":[{"id":"https://openalex.org/I134738993","display_name":"Shandong University of Traditional Chinese Medicine","ror":"https://ror.org/0523y5c19","country_code":"CN","type":"education","lineage":["https://openalex.org/I134738993"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yanhong Yu","raw_affiliation_strings":["College of Traditional Chinese Medicine, Shandong University of Traditional Chinese Medicine, Jinan, China","Shandong University of Traditional Chinese Medicine, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0002-6547-6320","affiliations":[{"raw_affiliation_string":"College of Traditional Chinese Medicine, Shandong University of Traditional Chinese Medicine, Jinan, China","institution_ids":["https://openalex.org/I134738993"]},{"raw_affiliation_string":"Shandong University of Traditional Chinese Medicine, Jinan, China","institution_ids":["https://openalex.org/I134738993"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Lin Lu","orcid":"https://orcid.org/0009-0006-6194-8577"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]},{"id":"https://openalex.org/I4210113117","display_name":"Shandong Mental Health Center","ror":"https://ror.org/024x8v141","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210113117"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin Lu","raw_affiliation_strings":["Shandong Mental Health Center, Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0009-0006-6194-8577","affiliations":[{"raw_affiliation_string":"Shandong Mental Health Center, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I4210113117","https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hao Wang","orcid":"https://orcid.org/0009-0003-9861-5993"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]},{"id":"https://openalex.org/I4210113117","display_name":"Shandong Mental Health Center","ror":"https://ror.org/024x8v141","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210113117"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Wang","raw_affiliation_strings":["Shandong Mental Health Center, Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0009-0003-9861-5993","affiliations":[{"raw_affiliation_string":"Shandong Mental Health Center, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I4210113117","https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021617086","display_name":"Yunshao Zheng","orcid":"https://orcid.org/0009-0008-0076-2968"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]},{"id":"https://openalex.org/I4210113117","display_name":"Shandong Mental Health Center","ror":"https://ror.org/024x8v141","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210113117"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunshao Zheng","raw_affiliation_strings":["Shandong Mental Health Center, Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0009-0008-0076-2968","affiliations":[{"raw_affiliation_string":"Shandong Mental Health Center, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I4210113117","https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104668625","display_name":"Yang Liu","orcid":"https://orcid.org/0009-0005-0759-2728"},"institutions":[{"id":"https://openalex.org/I117532281","display_name":"Guangzhou University of Chinese Medicine","ror":"https://ror.org/03qb7bg95","country_code":"CN","type":"education","lineage":["https://openalex.org/I117532281"]},{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]},{"id":"https://openalex.org/I4210131162","display_name":"Fifth Hospital In Wuhan","ror":"https://ror.org/02hpna219","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210131162"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Endocrinology, The Fifth Clinical College, Guangzhou, China","Guangzhou University of Chinese Medicine, Guangzhou, China"],"raw_orcid":"https://orcid.org/0009-0005-0759-2728","affiliations":[{"raw_affiliation_string":"Endocrinology, The Fifth Clinical College, Guangzhou, China","institution_ids":["https://openalex.org/I4210131162","https://openalex.org/I37461747"]},{"raw_affiliation_string":"Guangzhou University of Chinese Medicine, Guangzhou, China","institution_ids":["https://openalex.org/I117532281"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101413909","display_name":"Qingxiang Wang","orcid":"https://orcid.org/0000-0002-8159-7739"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]},{"id":"https://openalex.org/I4210113117","display_name":"Shandong Mental Health Center","ror":"https://ror.org/024x8v141","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210113117"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingxiang Wang","raw_affiliation_strings":["Shandong Mental Health Center, Shandong University, Jinan, China"],"raw_orcid":"https://orcid.org/0000-0002-8159-7739","affiliations":[{"raw_affiliation_string":"Shandong Mental Health Center, Shandong University, Jinan, China","institution_ids":["https://openalex.org/I4210113117","https://openalex.org/I154099455"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.3436,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.97616276,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"35","issue":"3","first_page":"2087","last_page":"2100"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9319000244140625,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9319000244140625,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.6221901178359985},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5665659308433533},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.5093561410903931},{"id":"https://openalex.org/keywords/facial-recognition-system","display_name":"Facial recognition system","score":0.45373550057411194},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42909806966781616},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3899621069431305},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3646244406700134}],"concepts":[{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.6221901178359985},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5665659308433533},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.5093561410903931},{"id":"https://openalex.org/C31510193","wikidata":"https://www.wikidata.org/wiki/Q1192553","display_name":"Facial recognition system","level":3,"score":0.45373550057411194},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42909806966781616},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3899621069431305},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3646244406700134}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcsvt.2024.3491098","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcsvt.2024.3491098","pdf_url":null,"source":{"id":"https://openalex.org/S115173108","display_name":"IEEE Transactions on Circuits and Systems for Video Technology","issn_l":"1051-8215","issn":["1051-8215","1558-2205"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Circuits and Systems for Video Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality","score":0.6100000143051147}],"awards":[{"id":"https://openalex.org/G267681543","display_name":null,"funder_award_id":"ZR2021MF079","funder_id":"https://openalex.org/F4320336925","funder_display_name":"Shandong Provincial Postdoctoral Science Foundation"},{"id":"https://openalex.org/G3316096639","display_name":null,"funder_award_id":"2021SFGC0504","funder_id":"https://openalex.org/F4320333596","funder_display_name":"Key Technology Research and Development Program of Shandong"},{"id":"https://openalex.org/G3342431209","display_name":null,"funder_award_id":"2020CXGC010901","funder_id":"https://openalex.org/F4320333596","funder_display_name":"Key Technology Research and Development Program of Shandong"},{"id":"https://openalex.org/G4048929270","display_name":null,"funder_award_id":"81573829","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6116303166","display_name":null,"funder_award_id":"2021SFGC0504","funder_id":"https://openalex.org/F4320336742","funder_display_name":"Key Research and Development Program of Liaoning Province"},{"id":"https://openalex.org/G6678275123","display_name":null,"funder_award_id":"ZR2021MF079","funder_id":"https://openalex.org/F4320324174","funder_display_name":"Natural Science Foundation of Shandong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320324174","display_name":"Natural Science Foundation of Shandong Province","ror":null},{"id":"https://openalex.org/F4320333596","display_name":"Key Technology Research and Development Program of Shandong","ror":null},{"id":"https://openalex.org/F4320336742","display_name":"Key Research and Development Program of Liaoning Province","ror":null},{"id":"https://openalex.org/F4320336925","display_name":"Shandong Provincial Postdoctoral Science Foundation","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W204854131","https://openalex.org/W1501669607","https://openalex.org/W1506726456","https://openalex.org/W1613418177","https://openalex.org/W1971813937","https://openalex.org/W1974515627","https://openalex.org/W1980331490","https://openalex.org/W1991060033","https://openalex.org/W2035505019","https://openalex.org/W2056403322","https://openalex.org/W2085662862","https://openalex.org/W2102413207","https://openalex.org/W2114613490","https://openalex.org/W2351382423","https://openalex.org/W2394851080","https://openalex.org/W2530421149","https://openalex.org/W2550143307","https://openalex.org/W2604847698","https://openalex.org/W2605671917","https://openalex.org/W2703895418","https://openalex.org/W2748115049","https://openalex.org/W2767088461","https://openalex.org/W2800833042","https://openalex.org/W2807126412","https://openalex.org/W2893536541","https://openalex.org/W2963566548","https://openalex.org/W2970981431","https://openalex.org/W2988226917","https://openalex.org/W3091476632","https://openalex.org/W3094259155","https://openalex.org/W3100219394","https://openalex.org/W3153807302","https://openalex.org/W3155375169","https://openalex.org/W3155665398","https://openalex.org/W3177318507","https://openalex.org/W3197055506","https://openalex.org/W3197558301","https://openalex.org/W3205969292","https://openalex.org/W4210598935","https://openalex.org/W4210766162","https://openalex.org/W4223499953","https://openalex.org/W4285132306","https://openalex.org/W4285188301","https://openalex.org/W4286370724","https://openalex.org/W4288050061","https://openalex.org/W4294811355","https://openalex.org/W4319988294","https://openalex.org/W4360897382","https://openalex.org/W4382240798","https://openalex.org/W4385245566","https://openalex.org/W4390190649","https://openalex.org/W4390605218","https://openalex.org/W6780221082","https://openalex.org/W6803376173","https://openalex.org/W7033056612"],"related_works":["https://openalex.org/W2123478443","https://openalex.org/W2115635058","https://openalex.org/W2088830394","https://openalex.org/W2081765545","https://openalex.org/W2088050694","https://openalex.org/W4319336808","https://openalex.org/W2153939644","https://openalex.org/W2116423617","https://openalex.org/W2347824352","https://openalex.org/W2112875849"],"abstract_inverted_index":{"Clinical":[0],"research":[1],"has":[2],"demonstrated":[3],"that":[4,188],"exploring":[5],"behavioral":[6],"signal":[7],"differences":[8],"between":[9],"depressed":[10,189,222],"patients":[11,190,223],"and":[12,38,44,75,115,151,201,219,242],"non-depressed":[13],"people":[14],"using":[15],"audiovisual":[16],"technology":[17],"is":[18,170],"an":[19,32,147,152],"effective":[20],"approach":[21],"for":[22,46],"achieving":[23,146],"depression":[24,47,55,132,159],"recognition.":[25,48,133],"Hence,":[26],"in":[27,102,144],"this":[28],"paper":[29],"we":[30,52,109,186],"propose":[31,53,110],"emotion":[33,72],"word":[34],"reading":[35],"experiment":[36],"(EWRE),":[37],"extract":[39],"features":[40],"from":[41],"facial":[42],"expressions":[43],"audios":[45],"Building":[49],"upon":[50],"this,":[51],"a":[54,67,87,111,116,225],"recognition":[56,160],"model":[57],"(DEP-Former),":[58],"which":[59],"deeply":[60],"integrates":[61],"multimodal":[62,79,112,138],"features.":[63],"DEP-Former":[64,140],"first":[65],"designs":[66],"modality":[68],"adapter":[69],"to":[70,123,194,208,212,233],"achieve":[71,124],"space":[73],"mapping":[74],"the":[76,94,162,173,178,182],"sharing":[77],"of":[78,89,96,127,149,155,181,227],"features,":[80],"addressing":[81],"cross-modal":[82],"inconsistencies.":[83],"Simultaneously,":[84],"it":[85],"proposes":[86],"mechanism":[88],"attention":[90,179],"index":[91],"sharing,":[92],"exceeding":[93],"limitations":[95],"cognitive":[97],"subjectivity":[98],"by":[99],"calculating":[100],"confidence":[101],"key":[103],"emotional":[104,196,214,236],"information":[105],"across":[106],"modalities.":[107],"Finally,":[108],"cross-attention":[113],"module":[114,122],"Bernoulli":[117],"distribution":[118],"feature":[119],"fusion":[120],"prediction":[121],"deep":[125],"integration":[126],"multilevel":[128],"information,":[129],"thereby":[130],"enabling":[131],"Compared":[134],"with":[135],"existing":[136],"advanced":[137],"models,":[139],"demonstrates":[141],"superior":[142],"performance":[143],"EWRE,":[145],"accuracy":[148],"0.9500":[150],"F1":[153],"score":[154],"0.9499,":[156],"significantly":[157],"enhancing":[158],"over":[161],"single-modality":[163],"methods.":[164],"Furthermore,":[165],"its":[166],"robust":[167],"generalization":[168],"ability":[169],"validated":[171],"on":[172],"AVEC":[174],"2014":[175],"dataset.":[176],"Through":[177],"query":[180],"interpretability":[183],"analysis":[184],"module,":[185],"discover":[187],"exhibit":[191,224],"heightened":[192],"sensitivity":[193,232],"negative":[195],"words,":[197,215],"such":[198],"as":[199,238],"dismissal":[200],"tragedy.":[202],"In":[203],"contrast,":[204],"healthy":[205],"individuals":[206],"tend":[207],"be":[209],"more":[210],"attuned":[211],"positive":[213,235],"including":[216],"passion,":[217],"purity,":[218],"justice.":[220],"Additionally,":[221],"degree":[226],"psychological":[228],"state":[229],"diversity,":[230],"showing":[231],"some":[234],"words":[237],"well.":[239],"Our":[240],"codes":[241],"data":[243],"are":[244],"available":[245],"at":[246],"<uri":[247],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[248],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">https://github.com/QLUTEmoTechCrew/DEP-Former</uri>.":[249]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":16}],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-10-10T00:00:00"}