{"id":"https://openalex.org/W4391128508","doi":"https://doi.org/10.1109/tcds.2024.3357618","title":"<i>Husformer</i>: A Multimodal Transformer for Multimodal Human State Recognition","display_name":"<i>Husformer</i>: A Multimodal Transformer for Multimodal Human State Recognition","publication_year":2024,"publication_date":"2024-01-23","ids":{"openalex":"https://openalex.org/W4391128508","doi":"https://doi.org/10.1109/tcds.2024.3357618"},"language":"en","primary_location":{"id":"doi:10.1109/tcds.2024.3357618","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2024.3357618","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023498177","display_name":"Ruiqi Wang","orcid":"https://orcid.org/0000-0002-4323-8989"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ruiqi Wang","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000372970","display_name":"Wonse Jo","orcid":"https://orcid.org/0000-0002-6904-5878"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wonse Jo","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101254875","display_name":"Dezhong Zhao","orcid":"https://orcid.org/0000-0003-4478-9595"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dezhong Zhao","raw_affiliation_strings":["College of Mechanical and Electrical Engineering, Beijing University of Chemical Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"College of Mechanical and Electrical Engineering, Beijing University of Chemical Technology, Beijing, China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107140891","display_name":"Weizheng Wang","orcid":"https://orcid.org/0000-0002-0637-4789"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Weizheng Wang","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101939654","display_name":"Arjun Gupte","orcid":"https://orcid.org/0009-0009-6414-2685"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arjun Gupte","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089355143","display_name":"Baijian Yang","orcid":"https://orcid.org/0000-0003-4440-3701"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Baijian Yang","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107143571","display_name":"Guohua Chen","orcid":"https://orcid.org/0000-0003-4167-467X"},"institutions":[{"id":"https://openalex.org/I75390827","display_name":"Beijing University of Chemical Technology","ror":"https://ror.org/00df5yc52","country_code":"CN","type":"education","lineage":["https://openalex.org/I75390827"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guohua Chen","raw_affiliation_strings":["College of Mechanical and Electrical Engineering, Beijing University of Chemical Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"College of Mechanical and Electrical Engineering, Beijing University of Chemical Technology, Beijing, China","institution_ids":["https://openalex.org/I75390827"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5076467173","display_name":"Byung\u2010Cheol Min","orcid":"https://orcid.org/0000-0001-6458-4365"},"institutions":[{"id":"https://openalex.org/I219193219","display_name":"Purdue University West Lafayette","ror":"https://ror.org/02dqehb95","country_code":"US","type":"education","lineage":["https://openalex.org/I219193219"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Byung-Cheol Min","raw_affiliation_strings":["Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer and Information Technology, Purdue University, West Lafayette, IN, USA","institution_ids":["https://openalex.org/I219193219"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5023498177"],"corresponding_institution_ids":["https://openalex.org/I219193219"],"apc_list":null,"apc_paid":null,"fwci":7.9709,"has_fulltext":false,"cited_by_count":45,"citation_normalized_percentile":{"value":0.98412967,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"16","issue":"4","first_page":"1374","last_page":"1390"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12740","display_name":"Gait Recognition and Analysis","score":0.9217000007629395,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12740","display_name":"Gait Recognition and Analysis","score":0.9217000007629395,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.7848248481750488},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7813441753387451},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6120193600654602},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.36536669731140137},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.2293841540813446},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.16250291466712952},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08178016543388367}],"concepts":[{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.7848248481750488},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7813441753387451},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6120193600654602},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.36536669731140137},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.2293841540813446},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.16250291466712952},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08178016543388367},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tcds.2024.3357618","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcds.2024.3357618","pdf_url":null,"source":{"id":"https://openalex.org/S2488537894","display_name":"IEEE Transactions on Cognitive and Developmental Systems","issn_l":"2379-8920","issn":["2379-8920","2379-8939"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cognitive and Developmental Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7683411322","display_name":null,"funder_award_id":"IIS-1846221","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":59,"referenced_works":["https://openalex.org/W1492815834","https://openalex.org/W1539811621","https://openalex.org/W1983830994","https://openalex.org/W2002055708","https://openalex.org/W2065073464","https://openalex.org/W2089394015","https://openalex.org/W2112290235","https://openalex.org/W2116341502","https://openalex.org/W2122410182","https://openalex.org/W2144961120","https://openalex.org/W2255466643","https://openalex.org/W2314395941","https://openalex.org/W2733014478","https://openalex.org/W2755014019","https://openalex.org/W2761964548","https://openalex.org/W2766825571","https://openalex.org/W2786768213","https://openalex.org/W2790404832","https://openalex.org/W2791957742","https://openalex.org/W2807304670","https://openalex.org/W2894771803","https://openalex.org/W2896220595","https://openalex.org/W2901809396","https://openalex.org/W2963351448","https://openalex.org/W2963781023","https://openalex.org/W2964051877","https://openalex.org/W2975376327","https://openalex.org/W2981372722","https://openalex.org/W3008191852","https://openalex.org/W3024961463","https://openalex.org/W3033146535","https://openalex.org/W3034296212","https://openalex.org/W3089475900","https://openalex.org/W3127637041","https://openalex.org/W3143320354","https://openalex.org/W3154396742","https://openalex.org/W3159301005","https://openalex.org/W3164305153","https://openalex.org/W3167354871","https://openalex.org/W3175508462","https://openalex.org/W3187148241","https://openalex.org/W4210907107","https://openalex.org/W4212863985","https://openalex.org/W4226393362","https://openalex.org/W4251598436","https://openalex.org/W4280564710","https://openalex.org/W4285044982","https://openalex.org/W4294982756","https://openalex.org/W4309568943","https://openalex.org/W4313164293","https://openalex.org/W4313590743","https://openalex.org/W4330337810","https://openalex.org/W4385245566","https://openalex.org/W4386071994","https://openalex.org/W4389665973","https://openalex.org/W4399619856","https://openalex.org/W6631216910","https://openalex.org/W6686207219","https://openalex.org/W6851211609"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W1596801655","https://openalex.org/W2359140296"],"abstract_inverted_index":{"Human":[0],"state":[1,89],"recognition":[2,34,90,169],"is":[3],"a":[4,29,135,187,191],"critical":[5],"topic":[6],"with":[7,197],"pervasive":[8],"and":[9,69,73,155,157,163,183,221],"important":[10],"applications":[11],"in":[12,48,60,116,143,167,214],"human-machine":[13],"systems.":[14],"Multi-modal":[15],"fusion,":[16],"which":[17,102],"entails":[18],"integrating":[19],"metrics":[20],"from":[21],"various":[22],"data":[23],"sources,":[24],"has":[25],"proven":[26],"to":[27,106,112,119,138,207],"be":[28],"potent":[30],"method":[31],"for":[32,55,86],"boosting":[33],"performance.":[35],"Although":[36],"recent":[37],"multi-modal-based":[38],"models":[39],"have":[40],"shown":[41],"promising":[42],"results,":[43],"they":[44,65],"often":[45],"fall":[46],"short":[47],"fully":[49],"leveraging":[50],"sophisticated":[51],"fusion":[52,62,145],"strategies":[53],"essential":[54],"modeling":[56],"adequate":[57],"cross-modal":[58,100,129],"dependencies":[59],"the":[61,128,144,168,171,184,209],"representation.":[63,146],"Instead,":[64],"rely":[66],"on":[67,149],"costly":[68],"inconsistent":[70],"feature":[71],"crafting":[72],"alignment.":[74],"To":[75],"address":[76],"this":[77],"limitation,":[78],"we":[79,97,133],"propose":[80,98],"an":[81,204],"end-toend":[82],"multi-modal":[83,87,181,199],"transformer":[84,137],"framework":[85],"human":[88,151,172],"called":[91],"<italic":[92,175,215],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[93,176,216,228],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Husformer</i>":[94,177,217],".":[95,218,230],"Specifically,":[96],"using":[99],"transformers,":[101],"inspire":[103],"one":[104],"modality":[105,189],"reinforce":[107],"itself":[108],"through":[109],"directly":[110],"attending":[111],"latent":[113],"relevance":[114],"revealed":[115],"other":[117],"modalities,":[118],"fuse":[120],"different":[121],"modalities":[122],"while":[123],"ensuring":[124],"sufficient":[125],"awareness":[126],"of":[127,170,186,211],"interactions":[130],"introduced.":[131],"Subsequently,":[132],"utilize":[134],"self-attention":[136],"further":[139],"prioritize":[140],"contextual":[141],"information":[142],"Extensive":[147],"experiments":[148],"two":[150,158],"emotion":[152],"corpora":[153],"(DEAP":[154],"WESAD)":[156],"cognitive":[159],"load":[160],"datasets":[161],"(MOCAS":[162],"CogLoad)":[164],"demonstrate":[165],"that":[166],"state,":[173],"our":[174],"outperforms":[178],"both":[179],"state-of-the-art":[180],"baselines":[182],"use":[185],"single":[188],"by":[190],"large":[192],"margin,":[193],"especially":[194],"when":[195],"dealing":[196],"raw":[198],"features.":[200],"We":[201],"also":[202],"conducted":[203],"ablation":[205],"study":[206],"show":[208],"benefits":[210],"each":[212],"component":[213],"Experimental":[219],"details":[220],"source":[222],"code":[223],"are":[224],"available":[225],"at:":[226],"<uri":[227],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">https://github.com/SMARTlab-Purdue/Husformer</uri>":[229]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":27},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2024-01-24T00:00:00"}