{"id":"https://openalex.org/W2896463661","doi":"https://doi.org/10.1145/3240508.3240714","title":"Human Conversation Analysis Using Attentive Multimodal Networks with Hierarchical Encoder-Decoder","display_name":"Human Conversation Analysis Using Attentive Multimodal Networks with Hierarchical Encoder-Decoder","publication_year":2018,"publication_date":"2018-10-15","ids":{"openalex":"https://openalex.org/W2896463661","doi":"https://doi.org/10.1145/3240508.3240714","mag":"2896463661","pmid":"https://pubmed.ncbi.nlm.nih.gov/32201865"},"language":"en","primary_location":{"id":"doi:10.1145/3240508.3240714","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3240508.3240714","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM international conference on Multimedia","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050492878","display_name":"Yue Gu","orcid":"https://orcid.org/0000-0002-6402-2386"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yue Gu","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100406079","display_name":"Xinyu Li","orcid":"https://orcid.org/0000-0001-5398-3707"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]},{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xinyu Li","raw_affiliation_strings":["Rutgers University & Amazon Inc., Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University & Amazon Inc., Piscataway, NJ, USA","institution_ids":["https://openalex.org/I1311688040","https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022463981","display_name":"Kaixiang Huang","orcid":"https://orcid.org/0000-0002-1256-153X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kaixiang Huang","raw_affiliation_strings":["Meitu Inc. & Rutgers University, Xiamen, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Meitu Inc. & Rutgers University, Xiamen, China","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005478056","display_name":"Shiyu Fu","orcid":"https://orcid.org/0009-0003-0061-0765"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shiyu Fu","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021980339","display_name":"Kangning Yang","orcid":"https://orcid.org/0000-0002-7106-0022"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kangning Yang","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010187139","display_name":"Shuhong Chen","orcid":"https://orcid.org/0000-0002-6120-6358"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shuhong Chen","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020750021","display_name":"Moliang Zhou","orcid":null},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Moliang Zhou","raw_affiliation_strings":["Amazon Inc., Boston, MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Inc., Boston, MA, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068126329","display_name":"Ivan Marsic","orcid":"https://orcid.org/0000-0002-1033-6865"},"institutions":[{"id":"https://openalex.org/I102322142","display_name":"Rutgers, The State University of New Jersey","ror":"https://ror.org/05vt9qd57","country_code":"US","type":"education","lineage":["https://openalex.org/I102322142"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ivan Marsic","raw_affiliation_strings":["Rutgers University, Piscataway, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rutgers University, Piscataway, NJ, USA","institution_ids":["https://openalex.org/I102322142"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.3794,"has_fulltext":false,"cited_by_count":36,"citation_normalized_percentile":{"value":0.94605562,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"2018","issue":null,"first_page":"537","last_page":"545"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10667","display_name":"Emotion and Mood Recognition","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/3205","display_name":"Experimental and Cognitive Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.812541127204895},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.6579248905181885},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.6544988751411438},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5804619193077087},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5019209384918213},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.49166256189346313},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.4677291810512543},{"id":"https://openalex.org/keywords/decoding-methods","display_name":"Decoding methods","score":0.4672308564186096},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.45463815331459045},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.433675080537796},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4305903911590576},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.39626526832580566},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.350294828414917},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.10412922501564026}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.812541127204895},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.6579248905181885},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.6544988751411438},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5804619193077087},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5019209384918213},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.49166256189346313},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.4677291810512543},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.4672308564186096},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.45463815331459045},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.433675080537796},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4305903911590576},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.39626526832580566},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.350294828414917},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.10412922501564026},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3240508.3240714","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3240508.3240714","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 26th ACM international conference on Multimedia","raw_type":"proceedings-article"},{"id":"pmid:32201865","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32201865","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ... ACM International Conference on Multimedia, with co-located Symposium & Workshops. ACM International Conference on Multimedia","raw_type":null},{"id":"pmh:oai:europepmc.org:6037021","is_oa":false,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7085889","pdf_url":null,"source":{"id":"https://openalex.org/S4306400806","display_name":"Europe PMC (PubMed Central)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1303153112","host_organization_name":"European Bioinformatics Institute","host_organization_lineage":["https://openalex.org/I1303153112"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6000000238418579,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1287356885","display_name":null,"funder_award_id":"R01 LM011834","funder_id":"https://openalex.org/F4320337372","funder_display_name":"U.S. National Library of Medicine"},{"id":"https://openalex.org/G6118814437","display_name":null,"funder_award_id":"R01LM011834","funder_id":"https://openalex.org/F4320332161","funder_display_name":"National Institutes of Health"}],"funders":[{"id":"https://openalex.org/F4320328861","display_name":"Ministry of Urban Development","ror":null},{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337372","display_name":"U.S. National Library of Medicine","ror":"https://ror.org/0060t0j89"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W1964744287","https://openalex.org/W1973270182","https://openalex.org/W1995562189","https://openalex.org/W2061116763","https://openalex.org/W2075398069","https://openalex.org/W2079725295","https://openalex.org/W2085662862","https://openalex.org/W2094244309","https://openalex.org/W2095176743","https://openalex.org/W2101534792","https://openalex.org/W2111645492","https://openalex.org/W2119821739","https://openalex.org/W2122563357","https://openalex.org/W2128160875","https://openalex.org/W2153579005","https://openalex.org/W2163605009","https://openalex.org/W2251394420","https://openalex.org/W2290180618","https://openalex.org/W2302255633","https://openalex.org/W2470673105","https://openalex.org/W2519656895","https://openalex.org/W2533262878","https://openalex.org/W2546919788","https://openalex.org/W2550557083","https://openalex.org/W2583643061","https://openalex.org/W2625297138","https://openalex.org/W2703895418","https://openalex.org/W2740550900","https://openalex.org/W2766969100","https://openalex.org/W2767249564","https://openalex.org/W2772633765","https://openalex.org/W2788586383","https://openalex.org/W2911964244","https://openalex.org/W2963377935","https://openalex.org/W2963495494","https://openalex.org/W2963710346","https://openalex.org/W2963785564","https://openalex.org/W2964010806","https://openalex.org/W2964266095","https://openalex.org/W2964308564","https://openalex.org/W4301142835"],"related_works":["https://openalex.org/W2905433371","https://openalex.org/W2888392564","https://openalex.org/W4310278675","https://openalex.org/W4388422664","https://openalex.org/W4390569940","https://openalex.org/W4361193272","https://openalex.org/W2963326959","https://openalex.org/W4388685194","https://openalex.org/W4312407344","https://openalex.org/W2894289927"],"abstract_inverted_index":{"Human":[0],"conversation":[1,31],"analysis":[2,93],"is":[3,56],"challenging":[4],"because":[5],"the":[6,132,142,150],"meaning":[7],"can":[8],"be":[9],"expressed":[10],"through":[11],"words,":[12],"intonation,":[13],"or":[14],"even":[15],"body":[16],"language":[17],"and":[18,42,90,104,145,157],"facial":[19],"expression.":[20],"We":[21,80,110,123],"introduce":[22,72],"a":[23,73],"hierarchical":[24,34,54],"encoder-decoder":[25],"structure":[26],"with":[27,77],"attention":[28,147,152],"mechanism":[29,153],"for":[30],"analysis.":[32],"The":[33,52],"encoder":[35],"learns":[36],"word-level":[37],"features":[38],"from":[39],"video,":[40],"audio,":[41],"text":[43],"data":[44],"that":[45,149],"are":[46],"then":[47],"formulated":[48],"into":[49],"conversation-level":[50],"features.":[51],"corresponding":[53],"decoder":[55],"able":[57],"to":[58],"predict":[59],"different":[60],"attributes":[61],"at":[62],"given":[63],"time":[64],"instances.":[65],"To":[66],"integrate":[67],"multiple":[68,137],"sensory":[69],"inputs,":[70],"we":[71],"novel":[74],"fusion":[75],"strategy":[76],"modality":[78,144],"attention.":[79],"evaluated":[81],"our":[82],"system":[83,96],"on":[84,107,118],"published":[85],"emotion":[86],"recognition,":[87],"sentiment":[88],"analysis,":[89],"speaker":[91],"trait":[92],"datasets.":[94,109,122],"Our":[95],"outperformed":[97,112],"previous":[98,113],"state-of-the-art":[99],"approaches":[100,114],"in":[101,115,127],"both":[102],"classification":[103],"regressions":[105],"tasks":[106],"three":[108],"also":[111],"generalization":[116],"tests":[117],"two":[119],"commonly":[120],"used":[121],"achieved":[124],"comparable":[125],"performance":[126],"predicting":[128],"co-existing":[129],"labels":[130],"using":[131],"proposed":[133,151],"model":[134,159],"instead":[135],"of":[136],"individual":[138],"models.":[139],"In":[140],"addition,":[141],"easily-visualized":[143],"temporal":[146],"demonstrated":[148],"helps":[154],"feature":[155],"selection":[156],"improves":[158],"interpretability.":[160]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":7},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":6}],"updated_date":"2026-06-24T13:16:06.693445","created_date":"2025-10-10T00:00:00"}