{"id":"https://openalex.org/W3094259155","doi":"https://doi.org/10.1109/taffc.2020.3031345","title":"Multimodal Spatiotemporal Representation for Automatic Depression Level Detection","display_name":"Multimodal Spatiotemporal Representation for Automatic Depression Level Detection","publication_year":2020,"publication_date":"2020-10-15","ids":{"openalex":"https://openalex.org/W3094259155","doi":"https://doi.org/10.1109/taffc.2020.3031345","mag":"3094259155"},"language":"en","primary_location":{"id":"doi:10.1109/taffc.2020.3031345","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2020.3031345","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5079696396","display_name":"Mingyue Niu","orcid":"https://orcid.org/0000-0002-4977-3202"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingyue Niu","raw_affiliation_strings":["National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences (UCAS), Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4977-3202","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Beijing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I19820366"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences (UCAS), Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112613657","display_name":"Jianhua Tao","orcid":"https://orcid.org/0000-0002-9344-6428"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210097554","display_name":"Center for Excellence in Brain Science and Intelligence Technology","ror":"https://ror.org/00vpwhm04","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210097554"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianhua Tao","raw_affiliation_strings":["National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences (UCAS), Beijing, China","CAS center for Excellence in Brain Science and Intelligence Technology, Bejing, China"],"raw_orcid":"https://orcid.org/0000-0002-9344-6428","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I19820366"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences (UCAS), Beijing, China","institution_ids":["https://openalex.org/I4210165038"]},{"raw_affiliation_string":"CAS center for Excellence in Brain Science and Intelligence Technology, Bejing, China","institution_ids":["https://openalex.org/I4210097554"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100395562","display_name":"Bin Liu","orcid":"https://orcid.org/0000-0003-1529-1552"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Liu","raw_affiliation_strings":["National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China"],"raw_orcid":"https://orcid.org/0000-0003-1529-1552","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100606274","display_name":"Jian Huang","orcid":"https://orcid.org/0000-0002-4638-2636"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Huang","raw_affiliation_strings":["National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China"],"raw_orcid":"https://orcid.org/0000-0002-4638-2636","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001973434","display_name":"Zheng Lian","orcid":"https://orcid.org/0000-0001-9477-0599"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng Lian","raw_affiliation_strings":["National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China"],"raw_orcid":"https://orcid.org/0000-0001-9477-0599","affiliations":[{"raw_affiliation_string":"National Laboratory of Pattern Recognition (NLPR), Institute of Automatic Chinese Academy of Sciences (CASIA), Bejing, China","institution_ids":["https://openalex.org/I4210112150","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":6.7953,"has_fulltext":false,"cited_by_count":155,"citation_normalized_percentile":{"value":0.97268585,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":"14","issue":"1","first_page":"294","last_page":"307"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12488","display_name":"Mental Health via Writing","score":0.929099977016449,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9236000180244446,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6728909015655518},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.6203330159187317},{"id":"https://openalex.org/keywords/pooling","display_name":"Pooling","score":0.5805708169937134},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.498319149017334},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4974205791950226},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.4508287310600281},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.43069326877593994},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3902684152126312}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6728909015655518},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.6203330159187317},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.5805708169937134},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.498319149017334},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4974205791950226},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.4508287310600281},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.43069326877593994},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3902684152126312},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/taffc.2020.3031345","is_oa":false,"landing_page_url":"https://doi.org/10.1109/taffc.2020.3031345","pdf_url":null,"source":{"id":"https://openalex.org/S104780363","display_name":"IEEE Transactions on Affective Computing","issn_l":"1949-3045","issn":["1949-3045","2371-9850"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Affective Computing","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5381013183","display_name":null,"funder_award_id":"61831022","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7264488544","display_name":"\u60c5\u611f\u8bed\u97f3\u97f3\u8272\u5206\u6790\u53ca\u8865\u507f\u65b9\u6cd5\u7814\u7a76","funder_award_id":"61771472","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8374701036","display_name":null,"funder_award_id":"61901473","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G953811167","display_name":null,"funder_award_id":"61773379","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W36756441","https://openalex.org/W1563348166","https://openalex.org/W1966268437","https://openalex.org/W1976066595","https://openalex.org/W1988088279","https://openalex.org/W1991060033","https://openalex.org/W1998512907","https://openalex.org/W1999839183","https://openalex.org/W2002725054","https://openalex.org/W2003502731","https://openalex.org/W2020163092","https://openalex.org/W2021549893","https://openalex.org/W2027192823","https://openalex.org/W2050705480","https://openalex.org/W2055134133","https://openalex.org/W2056403322","https://openalex.org/W2057563799","https://openalex.org/W2058158720","https://openalex.org/W2060779380","https://openalex.org/W2071268704","https://openalex.org/W2084242110","https://openalex.org/W2095540482","https://openalex.org/W2102505790","https://openalex.org/W2108223677","https://openalex.org/W2131042978","https://openalex.org/W2139916508","https://openalex.org/W2160947254","https://openalex.org/W2167854178","https://openalex.org/W2182295029","https://openalex.org/W2182998784","https://openalex.org/W2394998899","https://openalex.org/W2400979883","https://openalex.org/W2570412145","https://openalex.org/W2584561145","https://openalex.org/W2592018447","https://openalex.org/W2736814566","https://openalex.org/W2747833299","https://openalex.org/W2748115049","https://openalex.org/W2760537051","https://openalex.org/W2765292728","https://openalex.org/W2766211031","https://openalex.org/W2770976973","https://openalex.org/W2800833042","https://openalex.org/W2803017656","https://openalex.org/W2805409402","https://openalex.org/W2887068828","https://openalex.org/W2889847952","https://openalex.org/W2896134990","https://openalex.org/W2957148965","https://openalex.org/W2960672642","https://openalex.org/W2973178409","https://openalex.org/W2987559831","https://openalex.org/W2996751911","https://openalex.org/W2999794487","https://openalex.org/W3003850414","https://openalex.org/W3004404036","https://openalex.org/W4246193833","https://openalex.org/W6743224853","https://openalex.org/W6769962118"],"related_works":["https://openalex.org/W4205899436","https://openalex.org/W2462138927","https://openalex.org/W3111427900","https://openalex.org/W4302009627","https://openalex.org/W4212781403","https://openalex.org/W2953234277","https://openalex.org/W2626256601","https://openalex.org/W147410782","https://openalex.org/W4388989680","https://openalex.org/W2900413183"],"abstract_inverted_index":{"Physiological":[0],"studies":[1],"have":[2],"shown":[3],"that":[4],"there":[5],"are":[6],"some":[7],"differences":[8],"in":[9],"speech":[10,58],"and":[11,16,31,64,78,155,172],"facial":[12],"activities":[13],"between":[14],"depressive":[15],"healthy":[17],"individuals.":[18],"Based":[19],"on":[20,169],"this":[21,114],"fact,":[22],"we":[23,54],"propose":[24],"a":[25,32],"novel":[26],"spatio-temporal":[27],"attention":[28,34,82],"(STA)":[29],"network":[30],"multimodal":[33,42,144],"feature":[35,97],"fusion":[36],"(MAFF)":[37],"strategy":[38],"to":[39,91,122,134],"obtain":[40],"the":[41,49,57,69,76,87,101,104,110,117,124,130,138,143,153,158,170,177],"representation":[43,145],"of":[44,103,109,126,129,179],"depression":[45,51,92,165,174],"cues":[46],"for":[47,163],"predicting":[48],"individual":[50],"level.":[52],"Specifically,":[53],"first":[55],"divide":[56],"amplitude":[59],"spectrum/video":[60],"into":[61,68,137,157],"fixed-length":[62],"segments":[63,67],"input":[65],"these":[66],"STA":[70,111],"network,":[71],"which":[72],"not":[73],"only":[74],"integrates":[75],"spatial":[77],"temporal":[79],"information":[80,149],"through":[81],"mechanism,":[83],"but":[84],"also":[85],"emphasizes":[86],"audio/video":[88,95,131,139],"frames":[89],"related":[90],"detection.":[93],"The":[94],"segment-level":[96,132],"is":[98,150],"obtained":[99],"from":[100],"output":[102],"last":[105],"full":[106],"connection":[107],"layer":[108],"network.":[112],"Second,":[113],"article":[115],"employs":[116],"eigen":[118],"evolution":[119],"pooling":[120],"method":[121],"summarize":[123],"changes":[125],"each":[127],"dimension":[128],"features":[133],"aggregate":[135],"them":[136],"level":[140],"feature.":[141],"Third,":[142],"with":[146],"modal":[147],"complementary":[148],"generated":[151],"using":[152],"MAFF":[154],"inputs":[156],"support":[159],"vector":[160],"regression":[161],"predictor":[162],"estimating":[164],"severity.":[166],"Experimental":[167],"results":[168],"AVEC2013":[171],"AVEC2014":[173],"databases":[175],"illustrate":[176],"effectiveness":[178],"our":[180],"method.":[181]},"counts_by_year":[{"year":2026,"cited_by_count":15},{"year":2025,"cited_by_count":55},{"year":2024,"cited_by_count":47},{"year":2023,"cited_by_count":15},{"year":2022,"cited_by_count":15},{"year":2021,"cited_by_count":8}],"updated_date":"2026-06-19T17:40:00.097472","created_date":"2025-10-10T00:00:00"}