{"id":"https://openalex.org/W2621864722","doi":"https://doi.org/10.1109/tmm.2017.2713408","title":"Multimodal 2D+3D Facial Expression Recognition With Deep Fusion Convolutional Neural Network","display_name":"Multimodal 2D+3D Facial Expression Recognition With Deep Fusion Convolutional Neural Network","publication_year":2017,"publication_date":"2017-06-08","ids":{"openalex":"https://openalex.org/W2621864722","doi":"https://doi.org/10.1109/tmm.2017.2713408","mag":"2621864722"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2017.2713408","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2017.2713408","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102003771","display_name":"Huibin Li","orcid":"https://orcid.org/0000-0001-9980-0152"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huibin Li","raw_affiliation_strings":["Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-9980-0152","affiliations":[{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037419606","display_name":"Jian Sun","orcid":"https://orcid.org/0000-0001-7206-0641"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Sun","raw_affiliation_strings":["Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0001-7206-0641","affiliations":[{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113541537","display_name":"Zongben Xu","orcid":null},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zongben Xu","raw_affiliation_strings":["Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi\u2019an Jiaotong University, Xi\u2019an, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"Institute for Information and System Sciences, School of Mathematics and Statistics, Xi'an Jiaotong University, Xi'an, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100443527","display_name":"Liming Chen","orcid":"https://orcid.org/0000-0002-3654-9498"},"institutions":[{"id":"https://openalex.org/I112936343","display_name":"\u00c9cole Centrale de Lyon","ror":"https://ror.org/05s6rge65","country_code":"FR","type":"education","lineage":["https://openalex.org/I112936343","https://openalex.org/I203339264"]},{"id":"https://openalex.org/I4210155607","display_name":"Laboratoire d'Informatique en Images et Syst\u00e8mes d'Information","ror":"https://ror.org/04dv4he91","country_code":"FR","type":"facility","lineage":["https://openalex.org/I100532134","https://openalex.org/I112936343","https://openalex.org/I1294671590","https://openalex.org/I188626449","https://openalex.org/I203339264","https://openalex.org/I203339264","https://openalex.org/I203339264","https://openalex.org/I203339264","https://openalex.org/I4210155607","https://openalex.org/I48430043"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Liming Chen","raw_affiliation_strings":["LIRIS UMR 5205, Department of Mathematics and Informatics, Ecole Centrale de Lyon, Lyon, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LIRIS UMR 5205, Department of Mathematics and Informatics, Ecole Centrale de Lyon, Lyon, France","institution_ids":["https://openalex.org/I112936343","https://openalex.org/I4210155607"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":21.605,"has_fulltext":false,"cited_by_count":209,"citation_normalized_percentile":{"value":0.99669798,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"19","issue":"12","first_page":"2816","last_page":"2831"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10057","display_name":"Face and Expression Recognition","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9934999942779541,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.9477676153182983},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8203365802764893},{"id":"https://openalex.org/keywords/subnet","display_name":"Subnet","score":0.7989927530288696},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7987397313117981},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.7874519228935242},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.7191089987754822},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.7103595733642578},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.5537011623382568},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.5180535316467285},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5130780935287476}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.9477676153182983},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8203365802764893},{"id":"https://openalex.org/C21099817","wikidata":"https://www.wikidata.org/wiki/Q7631721","display_name":"Subnet","level":2,"score":0.7989927530288696},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7987397313117981},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.7874519228935242},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.7191089987754822},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.7103595733642578},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.5537011623382568},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.5180535316467285},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5130780935287476},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2017.2713408","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2017.2713408","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2614431373","display_name":"\u56fe\u50cf\u7ec6\u7c92\u5ea6\u8bc6\u522b\u7684\u663e\u8457\u6027\u7279\u5f81\u5b66\u4e60\u7b97\u6cd5\u7814\u7a76","funder_award_id":"61472313","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3272594795","display_name":"Serious Game for Children with autistic spectrum disorders based on Multimodal Emotion Imitation","funder_award_id":"ANR-13-CORD-0004","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G3455530712","display_name":null,"funder_award_id":"2014M560785","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G3773214607","display_name":"BIOmetrics spooF resistance Enhancement, evaluatioN and CErtification","funder_award_id":"ANR-13-INSE-0004","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G4246340258","display_name":null,"funder_award_id":"11401464","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4297193412","display_name":null,"funder_award_id":"11622106","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":82,"referenced_works":["https://openalex.org/W101526648","https://openalex.org/W171902450","https://openalex.org/W300533172","https://openalex.org/W825525975","https://openalex.org/W1040410175","https://openalex.org/W1480583224","https://openalex.org/W1557043124","https://openalex.org/W1566413196","https://openalex.org/W1603155784","https://openalex.org/W1967947242","https://openalex.org/W1968015059","https://openalex.org/W1974210421","https://openalex.org/W1978435065","https://openalex.org/W1981918162","https://openalex.org/W1997145183","https://openalex.org/W2006406015","https://openalex.org/W2006943008","https://openalex.org/W2006952256","https://openalex.org/W2011556862","https://openalex.org/W2013029619","https://openalex.org/W2018776244","https://openalex.org/W2021438259","https://openalex.org/W2030129920","https://openalex.org/W2034069713","https://openalex.org/W2051526232","https://openalex.org/W2051688810","https://openalex.org/W2062118960","https://openalex.org/W2065379720","https://openalex.org/W2066941820","https://openalex.org/W2068610869","https://openalex.org/W2069190036","https://openalex.org/W2072730064","https://openalex.org/W2082645628","https://openalex.org/W2083021723","https://openalex.org/W2086516372","https://openalex.org/W2095399814","https://openalex.org/W2102539767","https://openalex.org/W2102713935","https://openalex.org/W2104539097","https://openalex.org/W2106947945","https://openalex.org/W2109774206","https://openalex.org/W2116875963","https://openalex.org/W2124901121","https://openalex.org/W2134592850","https://openalex.org/W2136063698","https://openalex.org/W2137306662","https://openalex.org/W2139916508","https://openalex.org/W2143829622","https://openalex.org/W2149563102","https://openalex.org/W2150293569","https://openalex.org/W2151430047","https://openalex.org/W2155541015","https://openalex.org/W2159017231","https://openalex.org/W2163864141","https://openalex.org/W2164623278","https://openalex.org/W2166034132","https://openalex.org/W2174772062","https://openalex.org/W2187089797","https://openalex.org/W2191045312","https://openalex.org/W2195207531","https://openalex.org/W2198512331","https://openalex.org/W2246249023","https://openalex.org/W2294427751","https://openalex.org/W2301789900","https://openalex.org/W2326887180","https://openalex.org/W2339620988","https://openalex.org/W2345305417","https://openalex.org/W2534547303","https://openalex.org/W2963173190","https://openalex.org/W4294375521","https://openalex.org/W4294379094","https://openalex.org/W6604142973","https://openalex.org/W6623178044","https://openalex.org/W6636286539","https://openalex.org/W6648737282","https://openalex.org/W6675549610","https://openalex.org/W6679975153","https://openalex.org/W6680591342","https://openalex.org/W6682778277","https://openalex.org/W6684042233","https://openalex.org/W6687558509","https://openalex.org/W6728342743"],"related_works":["https://openalex.org/W2356206668","https://openalex.org/W2131631951","https://openalex.org/W2130707537","https://openalex.org/W2105155969","https://openalex.org/W3107204728","https://openalex.org/W2899027234","https://openalex.org/W3047363187","https://openalex.org/W3119773509","https://openalex.org/W3177373753","https://openalex.org/W4309346246"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"a":[3,22,26,31,79,131],"novel":[4],"and":[5,30,60,74,127,161,176,206],"efficient":[6],"deep":[7,103,156,159,201,207],"fusion":[8,28,75,128,210],"convolutional":[9],"neural":[10],"network":[11],"(DF-CNN)":[12],"for":[13,71,211],"multimodal":[14,212],"2D+3D":[15,213],"facial":[16,49,82],"expression":[17,114],"recognition":[18],"(FER).":[19],"DF-CNN":[20,70,123,182],"comprises":[21],"feature":[23,27,72,125],"extraction":[24],"subnet,":[25,29],"softmax":[32,109],"layer.":[33],"In":[34,179],"particular,":[35],"each":[36],"textured":[37],"three-dimensional":[38],"(3D)":[39],"face":[40,167],"scan":[41],"is":[42,87,195],"represented":[43],"as":[44],"six":[45],"types":[46],"of":[47,64,140,150,191,199],"2D":[48],"attribute":[50],"maps":[51],"(i.e.,":[52,169],"geometry":[53],"map,":[54,59],"three":[55,165],"normal":[56],"maps,":[57],"curvature":[58],"texture":[61],"map),":[62],"all":[63,180],"which":[65],"are":[66],"jointly":[67],"fed":[68],"into":[69,130],"learning":[73,93,126,129],"learning,":[76],"resulting":[77],"in":[78],"highly":[80],"concentrated":[81],"representation":[83],"(32-dimensional).":[84],"Expression":[85],"prediction":[86,110],"performed":[88],"by":[89],"two":[90],"ways:":[91],"1)":[92],"linear":[94],"support":[95],"vector":[96],"machine":[97],"classifiers":[98],"using":[99,111],"the":[100,112,138,148,185,189,196],"32-dimensional":[101],"fused":[102],"features,":[104,154,157,160],"or":[105],"2)":[106],"directly":[107],"performing":[108],"six-dimensional":[113],"probability":[115],"vectors.":[116],"Different":[117],"from":[118],"existing":[119],"3D":[120,166,204],"FER":[121,205],"methods,":[122],"combines":[124],"single":[132],"end-to-end":[133],"training":[134],"framework.":[135],"To":[136,188],"demonstrate":[137],"effectiveness":[139],"DF-CNN,":[141],"we":[142],"conducted":[143],"comprehensive":[144],"experiments":[145],"to":[146,203],"compare":[147],"performance":[149],"DFCNN":[151],"with":[152],"handcrafted":[153],"pre-trained":[155],"finetuned":[158],"state-of-the-art":[162],"methods":[163],"on":[164],"datasets":[168],"BU-3DFE":[170,173],"Subset":[171,174],"I,":[172],"II,":[175],"Bosphorus":[177],"Subset).":[178],"cases,":[181],"consistently":[183],"achieved":[184],"best":[186,190],"results.":[187],"our":[192],"knowledge,":[193],"this":[194],"first":[197],"work":[198],"introducing":[200],"CNN":[202],"learning-based":[208],"featurelevel":[209],"FER.":[214]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":27},{"year":2021,"cited_by_count":46},{"year":2020,"cited_by_count":34},{"year":2019,"cited_by_count":49},{"year":2018,"cited_by_count":13}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}