{"id":"https://openalex.org/W2002607236","doi":"https://doi.org/10.1109/fg.2011.5771372","title":"Real-time inference of mental states from facial expressions and upper body gestures","display_name":"Real-time inference of mental states from facial expressions and upper body gestures","publication_year":2011,"publication_date":"2011-03-01","ids":{"openalex":"https://openalex.org/W2002607236","doi":"https://doi.org/10.1109/fg.2011.5771372","mag":"2002607236"},"language":"en","primary_location":{"id":"doi:10.1109/fg.2011.5771372","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fg.2011.5771372","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Face and Gesture 2011","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://dspace.mit.edu/bitstream/1721.1/67458/1/FERA%202011_McDuff.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058610902","display_name":"Tadas Baltru\u0161aitis","orcid":"https://orcid.org/0000-0001-7923-8780"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Tadas Baltrusaitis","raw_affiliation_strings":["Computer Laboratory, University of Cambridge, Cambridge, UK","Computer Laboratory, University of Cambridge Cambridge UK"],"affiliations":[{"raw_affiliation_string":"Computer Laboratory, University of Cambridge, Cambridge, UK","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Computer Laboratory, University of Cambridge Cambridge UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100681741","display_name":"Daniel McDuff","orcid":"https://orcid.org/0000-0001-7313-0082"},"institutions":[{"id":"https://openalex.org/I4210122954","display_name":"MIT Lincoln Laboratory","ror":"https://ror.org/022z6jk58","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210122954","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel McDuff","raw_affiliation_strings":["MIT Media Laboratory, Cambridge, USA"],"affiliations":[{"raw_affiliation_string":"MIT Media Laboratory, Cambridge, USA","institution_ids":["https://openalex.org/I4210122954"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110284806","display_name":"Ntombikayise Banda","orcid":null},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ntombikayise Banda","raw_affiliation_strings":["Computer Laboratory, University of Cambridge, Cambridge, UK","Computer Laboratory, University of Cambridge Cambridge UK"],"affiliations":[{"raw_affiliation_string":"Computer Laboratory, University of Cambridge, Cambridge, UK","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Computer Laboratory, University of Cambridge Cambridge UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050992575","display_name":"Marwa Mahmoud","orcid":"https://orcid.org/0000-0002-5932-9113"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Marwa Mahmoud","raw_affiliation_strings":["Computer Laboratory, University of Cambridge, Cambridge, UK","Computer Laboratory, University of Cambridge Cambridge UK"],"affiliations":[{"raw_affiliation_string":"Computer Laboratory, University of Cambridge, Cambridge, UK","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Computer Laboratory, University of Cambridge Cambridge UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051143813","display_name":"Rana el Kaliouby","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122954","display_name":"MIT Lincoln Laboratory","ror":"https://ror.org/022z6jk58","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210122954","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rana el Kaliouby","raw_affiliation_strings":["MIT Media Laboratory, Cambridge, USA"],"affiliations":[{"raw_affiliation_string":"MIT Media Laboratory, Cambridge, USA","institution_ids":["https://openalex.org/I4210122954"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103077811","display_name":"Peter Robinson","orcid":"https://orcid.org/0000-0003-0347-3789"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Peter Robinson","raw_affiliation_strings":["Computer Laboratory, University of Cambridge, Cambridge, UK","Computer Laboratory, University of Cambridge Cambridge UK"],"affiliations":[{"raw_affiliation_string":"Computer Laboratory, University of Cambridge, Cambridge, UK","institution_ids":["https://openalex.org/I241749"]},{"raw_affiliation_string":"Computer Laboratory, University of Cambridge Cambridge UK","institution_ids":["https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5087366916","display_name":"Rosalind W. Picard","orcid":"https://orcid.org/0000-0002-5661-0022"},"institutions":[{"id":"https://openalex.org/I4210122954","display_name":"MIT Lincoln Laboratory","ror":"https://ror.org/022z6jk58","country_code":"US","type":"facility","lineage":["https://openalex.org/I4210122954","https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rosalind Picard","raw_affiliation_strings":["MIT Media Laboratory, Cambridge, USA"],"affiliations":[{"raw_affiliation_string":"MIT Media Laboratory, Cambridge, USA","institution_ids":["https://openalex.org/I4210122954"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5058610902"],"corresponding_institution_ids":["https://openalex.org/I241749"],"apc_list":null,"apc_paid":null,"fwci":6.1976,"has_fulltext":false,"cited_by_count":71,"citation_normalized_percentile":{"value":0.95947299,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"909","last_page":"914"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11707","display_name":"Gaze Tracking and Assistive Technology","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1709","display_name":"Human-Computer Interaction"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11448","display_name":"Face recognition and analysis","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sadness","display_name":"Sadness","score":0.879981517791748},{"id":"https://openalex.org/keywords/gesture","display_name":"Gesture","score":0.8350732326507568},{"id":"https://openalex.org/keywords/anger","display_name":"Anger","score":0.663127601146698},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6141963005065918},{"id":"https://openalex.org/keywords/facial-expression","display_name":"Facial expression","score":0.5662782788276672},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5381371378898621},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.533636212348938},{"id":"https://openalex.org/keywords/hidden-markov-model","display_name":"Hidden Markov model","score":0.51947420835495},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.49457094073295593},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.4881359040737152},{"id":"https://openalex.org/keywords/frame","display_name":"Frame (networking)","score":0.46778735518455505},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.46086299419403076},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.46039265394210815},{"id":"https://openalex.org/keywords/orientation","display_name":"Orientation (vector space)","score":0.41874799132347107},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.39015698432922363},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3395003378391266},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.29968032240867615},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.22961753606796265},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.07390540838241577}],"concepts":[{"id":"https://openalex.org/C2779812673","wikidata":"https://www.wikidata.org/wiki/Q169251","display_name":"Sadness","level":3,"score":0.879981517791748},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.8350732326507568},{"id":"https://openalex.org/C2779302386","wikidata":"https://www.wikidata.org/wiki/Q79871","display_name":"Anger","level":2,"score":0.663127601146698},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6141963005065918},{"id":"https://openalex.org/C195704467","wikidata":"https://www.wikidata.org/wiki/Q327968","display_name":"Facial expression","level":2,"score":0.5662782788276672},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5381371378898621},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.533636212348938},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.51947420835495},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.49457094073295593},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.4881359040737152},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.46778735518455505},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.46086299419403076},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.46039265394210815},{"id":"https://openalex.org/C16345878","wikidata":"https://www.wikidata.org/wiki/Q107472979","display_name":"Orientation (vector space)","level":2,"score":0.41874799132347107},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.39015698432922363},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3395003378391266},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.29968032240867615},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22961753606796265},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.07390540838241577},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C114793014","wikidata":"https://www.wikidata.org/wiki/Q52109","display_name":"Geomorphology","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/fg.2011.5771372","is_oa":false,"landing_page_url":"https://doi.org/10.1109/fg.2011.5771372","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Face and Gesture 2011","raw_type":"proceedings-article"},{"id":"pmh:oai:dspace.mit.edu:1721.1/67458","is_oa":true,"landing_page_url":"https://dspace.mit.edu/bitstream/1721.1/67458/1/FERA%202011_McDuff.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Javier Hernandez Rivera","raw_type":"http://purl.org/eprint/type/ConferencePaper"},{"id":"pmh:oai:eprints.gla.ac.uk:259889","is_oa":false,"landing_page_url":"http://eprints.gla.ac.uk/259889/","pdf_url":null,"source":{"id":"https://openalex.org/S4210235606","display_name":"ENLIGHTEN (Jurnal Bimbingan dan Konseling Islam)","issn_l":"2622-8912","issn":["2622-8912","2622-8920"],"is_oa":false,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Conference Proceedings"}],"best_oa_location":{"id":"pmh:oai:dspace.mit.edu:1721.1/67458","is_oa":true,"landing_page_url":"https://dspace.mit.edu/bitstream/1721.1/67458/1/FERA%202011_McDuff.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400425","display_name":"DSpace@MIT (Massachusetts Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63966007","host_organization_name":"Massachusetts Institute of Technology","host_organization_lineage":["https://openalex.org/I63966007"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Javier Hernandez Rivera","raw_type":"http://purl.org/eprint/type/ConferencePaper"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":17,"referenced_works":["https://openalex.org/W206006680","https://openalex.org/W414188911","https://openalex.org/W1555698685","https://openalex.org/W1837242107","https://openalex.org/W2038952578","https://openalex.org/W2104912728","https://openalex.org/W2108241350","https://openalex.org/W2119925967","https://openalex.org/W2129106196","https://openalex.org/W2133001582","https://openalex.org/W2140351517","https://openalex.org/W2171939880","https://openalex.org/W2173219554","https://openalex.org/W2296659146","https://openalex.org/W2912990735","https://openalex.org/W6633376576","https://openalex.org/W6638857374"],"related_works":["https://openalex.org/W2119012436","https://openalex.org/W3135546221","https://openalex.org/W2423081157","https://openalex.org/W2071993326","https://openalex.org/W2582288209","https://openalex.org/W2062652500","https://openalex.org/W2129567526","https://openalex.org/W4243434126","https://openalex.org/W4313639537","https://openalex.org/W3035983661"],"abstract_inverted_index":{"We":[0],"present":[1],"a":[2,68,93,128],"real-time":[3],"system":[4,24],"for":[5,102,109,139],"detecting":[6],"facial":[7,20,36],"action":[8,37,111],"units":[9,38,112],"and":[10,16,19,39,48,64],"inferring":[11],"emotional":[12,80,141],"states":[13,142],"from":[14,44],"head":[15,40,63],"shoulder":[17,65],"gestures":[18],"expressions.":[21],"The":[22,88,105,135],"dynamic":[23],"uses":[25],"three":[26],"levels":[27],"of":[28,60,85],"inference":[29],"on":[30,83,127],"progressively":[31],"longer":[32],"time":[33],"scales.":[34],"Firstly,":[35],"orientation":[41],"are":[42,55],"identified":[43],"22":[45],"feature":[46],"points":[47],"Gabor":[49],"filters.":[50],"Secondly,":[51],"Hidden":[52],"Markov":[53],"Models":[54],"used":[56,75],"to":[57,76],"classify":[58],"sequences":[59],"actions":[61],"into":[62],"gestures.":[66,87],"Finally,":[67],"multi":[69],"level":[70],"Dynamic":[71],"Bayesian":[72],"Network":[73],"is":[74,97],"model":[77],"the":[78,100,152,157],"unfolding":[79],"state":[81,91],"based":[82],"probabilities":[84],"different":[86],"most":[89],"probable":[90],"over":[92],"given":[94],"video":[95],"clip":[96],"chosen":[98],"as":[99],"label":[101],"that":[103],"clip.":[104],"average":[106,136],"F1":[107],"score":[108],"12":[110],"(AUs":[113],"1,":[114],"2,":[115],"4,":[116],"6,":[117],"7,":[118],"10,":[119],"12,":[120],"15,":[121],"17,":[122],"18,":[123],"25,":[124],"26),":[125],"labelled":[126],"frame":[129,131],"by":[130],"basis,":[132],"was":[133,148],"0.461.":[134],"classification":[137],"rate":[138],"five":[140],"(anger,":[143],"fear,":[144],"joy,":[145],"relief,":[146],"sadness)":[147],"0.440.":[149],"Sadness":[150],"had":[151],"greatest":[153],"rate,":[154],"0.64,":[155],"anger":[156],"smallest,":[158],"0.11.":[159]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":12},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":7},{"year":2015,"cited_by_count":7},{"year":2014,"cited_by_count":9},{"year":2013,"cited_by_count":5},{"year":2012,"cited_by_count":6}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}