{"id":"https://openalex.org/W4391021662","doi":"https://doi.org/10.1109/asru57964.2023.10389694","title":"Boosting Modality Representation With Pre-Trained Models and Multi-Task Training for Multimodal Sentiment Analysis","display_name":"Boosting Modality Representation With Pre-Trained Models and Multi-Task Training for Multimodal Sentiment Analysis","publication_year":2023,"publication_date":"2023-12-16","ids":{"openalex":"https://openalex.org/W4391021662","doi":"https://doi.org/10.1109/asru57964.2023.10389694"},"language":"en","primary_location":{"id":"doi:10.1109/asru57964.2023.10389694","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/asru57964.2023.10389694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5048171604","display_name":"Jiarui Hai","orcid":"https://orcid.org/0000-0001-9968-7372"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jiarui Hai","raw_affiliation_strings":["Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA"],"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101293967","display_name":"Yu-Jeh Liu","orcid":"https://orcid.org/0009-0007-7564-6198"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu-Jeh Liu","raw_affiliation_strings":["Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA"],"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5038788686","display_name":"Mounya Elhilali","orcid":"https://orcid.org/0000-0003-2597-738X"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mounya Elhilali","raw_affiliation_strings":["Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA"],"affiliations":[{"raw_affiliation_string":"Johns Hopkins University,Laboratory for Computational Auditory Perception,Baltimore,USA","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Laboratory for Computational Auditory Perception, Johns Hopkins University, Baltimore, USA","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5048171604"],"corresponding_institution_ids":["https://openalex.org/I145311948"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20322964,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"abs/1606.06259","issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8066743612289429},{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.7884203195571899},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.6906000375747681},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.6503618359565735},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"Sentiment analysis","score":0.6205464005470276},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.608997642993927},{"id":"https://openalex.org/keywords/multimodal-learning","display_name":"Multimodal learning","score":0.5563172101974487},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5177375674247742},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4655265212059021},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.46175193786621094},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.44877636432647705},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3830608129501343}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8066743612289429},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.7884203195571899},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.6906000375747681},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.6503618359565735},{"id":"https://openalex.org/C66402592","wikidata":"https://www.wikidata.org/wiki/Q2271421","display_name":"Sentiment analysis","level":2,"score":0.6205464005470276},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.608997642993927},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.5563172101974487},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5177375674247742},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4655265212059021},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.46175193786621094},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.44877636432647705},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3830608129501343},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/asru57964.2023.10389694","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/asru57964.2023.10389694","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2023 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.6299999952316284,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":41,"referenced_works":["https://openalex.org/W2026243162","https://openalex.org/W2055332436","https://openalex.org/W2061116763","https://openalex.org/W2122563357","https://openalex.org/W2191779130","https://openalex.org/W2395639500","https://openalex.org/W2465534249","https://openalex.org/W2619947201","https://openalex.org/W2807126412","https://openalex.org/W2896457183","https://openalex.org/W2958482160","https://openalex.org/W2963686995","https://openalex.org/W2963702064","https://openalex.org/W2963710346","https://openalex.org/W2964010806","https://openalex.org/W2964051877","https://openalex.org/W2964216663","https://openalex.org/W2979826702","https://openalex.org/W2999553663","https://openalex.org/W3034266838","https://openalex.org/W3034849760","https://openalex.org/W3093051361","https://openalex.org/W3096690837","https://openalex.org/W3103167052","https://openalex.org/W3206008172","https://openalex.org/W3206529771","https://openalex.org/W3209059054","https://openalex.org/W3211224152","https://openalex.org/W4221155339","https://openalex.org/W4224933771","https://openalex.org/W4297841872","https://openalex.org/W4307823382","https://openalex.org/W4308222497","https://openalex.org/W4312560592","https://openalex.org/W4376607994","https://openalex.org/W6719667659","https://openalex.org/W6755207826","https://openalex.org/W6755541679","https://openalex.org/W6803378298","https://openalex.org/W6955071965","https://openalex.org/W7073743773"],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W203959209","https://openalex.org/W2167701463","https://openalex.org/W2110287964","https://openalex.org/W4307407935"],"abstract_inverted_index":{"Sentiment":[0,56],"analysis":[1],"has":[2,12],"traditionally":[3],"leveraged":[4],"information":[5,32],"from":[6],"text":[7],"data.":[8],"More":[9],"recently,":[10],"it":[11],"become":[13],"increasingly":[14],"clear":[15],"that":[16],"multimodal":[17],"data":[18],"provides":[19],"a":[20,46],"rich":[21],"space":[22],"to":[23,50,76],"drastically":[24],"boost":[25],"interpretation":[26],"of":[27,71,83],"human":[28],"sentiments":[29],"by":[30],"harnessing":[31],"across":[33],"multiple":[34],"modalities.":[35,105],"In":[36],"this":[37],"study,":[38],"we":[39],"incorporate":[40],"pre-trained":[41],"feature":[42],"extractors":[43],"and":[44,103],"propose":[45],"multitask":[47],"training":[48],"strategy":[49],"improve":[51],"modality":[52],"representations":[53],"for":[54],"Multimodal":[55],"Analysis":[57],"(MSA).":[58],"The":[59],"experimental":[60],"results":[61],"on":[62,92],"the":[63,68,72,81],"CH-SIMS":[64],"v2":[65],"dataset":[66],"demonstrate":[67],"superior":[69],"performance":[70],"proposed":[73,85],"system":[74],"compared":[75],"existing":[77],"state-of-the-art":[78],"methods,":[79],"validating":[80],"effectiveness":[82],"our":[84,88],"approach.":[86],"Furthermore,":[87],"framework":[89],"reduces":[90],"reliance":[91],"textual":[93],"data,":[94],"achieving":[95],"competitive":[96],"outcomes":[97],"even":[98],"when":[99],"utilizing":[100],"only":[101],"auditory":[102],"visual":[104]},"counts_by_year":[],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-10-10T00:00:00"}