{"id":"https://openalex.org/W4403713273","doi":"https://doi.org/10.1145/3689062.3689084","title":"LLM-Driven Multimodal Fusion for Human Perception Analysis","display_name":"LLM-Driven Multimodal Fusion for Human Perception Analysis","publication_year":2024,"publication_date":"2024-10-23","ids":{"openalex":"https://openalex.org/W4403713273","doi":"https://doi.org/10.1145/3689062.3689084"},"language":"en","primary_location":{"id":"doi:10.1145/3689062.3689084","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689062.3689084","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th on Multimodal Sentiment Analysis Challenge and Workshop: Social Perception and Humor","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://oa.upm.es/84580/1/10259693.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094034370","display_name":"Sergio Esteban-Romero","orcid":"https://orcid.org/0009-0008-6336-7877"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":true,"raw_author_name":"Sergio Esteban-Romero","raw_affiliation_strings":["THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":"https://orcid.org/0009-0008-6336-7877","affiliations":[{"raw_affiliation_string":"THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093620465","display_name":"Iv\u00e1n Mart\u00edn-Fern\u00e1ndez","orcid":"https://orcid.org/0009-0004-2769-9752"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Iv\u00e1n Mart\u00edn-Fern\u00e1ndez","raw_affiliation_strings":["THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":"https://orcid.org/0009-0004-2769-9752","affiliations":[{"raw_affiliation_string":"THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058484271","display_name":"Manuel Gil-Mart\u00edn","orcid":"https://orcid.org/0000-0002-4285-6224"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Manuel Gil-Mart\u00edn","raw_affiliation_strings":["THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0002-4285-6224","affiliations":[{"raw_affiliation_string":"THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019802248","display_name":"David Griol","orcid":"https://orcid.org/0000-0001-6266-5321"},"institutions":[{"id":"https://openalex.org/I173304897","display_name":"Universidad de Granada","ror":"https://ror.org/04njjy449","country_code":"ES","type":"education","lineage":["https://openalex.org/I173304897"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"David Griol-Barres","raw_affiliation_strings":["Universidad de Granada, Granada, Spain"],"raw_orcid":"https://orcid.org/0000-0001-6266-5321","affiliations":[{"raw_affiliation_string":"Universidad de Granada, Granada, Spain","institution_ids":["https://openalex.org/I173304897"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057526407","display_name":"Zoraida Callejas","orcid":"https://orcid.org/0000-0001-8891-5237"},"institutions":[{"id":"https://openalex.org/I173304897","display_name":"Universidad de Granada","ror":"https://ror.org/04njjy449","country_code":"ES","type":"education","lineage":["https://openalex.org/I173304897"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Zoraida Callejas-Carri\u00f3n","raw_affiliation_strings":["Universidad de Granada, Granada, Spain"],"raw_orcid":"https://orcid.org/0000-0001-8891-5237","affiliations":[{"raw_affiliation_string":"Universidad de Granada, Granada, Spain","institution_ids":["https://openalex.org/I173304897"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5032875587","display_name":"Fernando Fern\u00e1ndez-Mart\u00ednez","orcid":"https://orcid.org/0000-0003-3877-0089"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Fernando Fern\u00e1ndez-Mart\u00ednez","raw_affiliation_strings":["THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0003-3877-0089","affiliations":[{"raw_affiliation_string":"THAU Group, IPTC, Universidad Polit\u00e9cnica de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I88060688"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5094034370"],"corresponding_institution_ids":["https://openalex.org/I88060688"],"apc_list":null,"apc_paid":null,"fwci":2.3179,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":{"value":0.90155465,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"45","last_page":"51"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9718999862670898,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9718999862670898,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9672999978065491,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12205","display_name":"Time Series Analysis and Forecasting","score":0.9584000110626221,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6131497025489807},{"id":"https://openalex.org/keywords/fusion","display_name":"Fusion","score":0.5869314670562744},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.5200552344322205},{"id":"https://openalex.org/keywords/sensor-fusion","display_name":"Sensor fusion","score":0.4323643743991852},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42499661445617676},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3640049397945404},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3523423671722412},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.15721845626831055},{"id":"https://openalex.org/keywords/neuroscience","display_name":"Neuroscience","score":0.06284701824188232}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6131497025489807},{"id":"https://openalex.org/C158525013","wikidata":"https://www.wikidata.org/wiki/Q2593739","display_name":"Fusion","level":2,"score":0.5869314670562744},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.5200552344322205},{"id":"https://openalex.org/C33954974","wikidata":"https://www.wikidata.org/wiki/Q486494","display_name":"Sensor fusion","level":2,"score":0.4323643743991852},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42499661445617676},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3640049397945404},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3523423671722412},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.15721845626831055},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.06284701824188232},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3689062.3689084","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3689062.3689084","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 5th on Multimodal Sentiment Analysis Challenge and Workshop: Social Perception and Humor","raw_type":"proceedings-article"},{"id":"pmh:oai:oa.upm.es:84580","is_oa":true,"landing_page_url":"https://oa.upm.es/84580/","pdf_url":"https://oa.upm.es/84580/1/10259693.pdf","source":{"id":"https://openalex.org/S4377196323","display_name":"UPM Digital Archive (Technical University of Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I88060688","host_organization_name":"Universidad Polit\u00e9cnica de Madrid","host_organization_lineage":["https://openalex.org/I88060688"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 5th on Multimodal Sentiment Analysis Challenge and Workshop: Social Perception and Humor | MuSe'24 | 28/10/2024-01/11/2024 | Melbourne, Australia","raw_type":"info:eu-repo/semantics/acceptedVersion"}],"best_oa_location":{"id":"pmh:oai:oa.upm.es:84580","is_oa":true,"landing_page_url":"https://oa.upm.es/84580/","pdf_url":"https://oa.upm.es/84580/1/10259693.pdf","source":{"id":"https://openalex.org/S4377196323","display_name":"UPM Digital Archive (Technical University of Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I88060688","host_organization_name":"Universidad Polit\u00e9cnica de Madrid","host_organization_lineage":["https://openalex.org/I88060688"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 5th on Multimodal Sentiment Analysis Challenge and Workshop: Social Perception and Humor | MuSe'24 | 28/10/2024-01/11/2024 | Melbourne, Australia","raw_type":"info:eu-repo/semantics/acceptedVersion"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5608557609","display_name":null,"funder_award_id":"101071191 ? HORIZON-EIC-2021-PATHFINDERCHALLENGES-01","funder_id":"https://openalex.org/F4320323817","funder_display_name":"Universitas Brawijaya"}],"funders":[{"id":"https://openalex.org/F4320323817","display_name":"Universitas Brawijaya","ror":"https://ror.org/01wk3d929"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403713273.pdf","grobid_xml":"https://content.openalex.org/works/W4403713273.grobid-xml"},"referenced_works_count":5,"referenced_works":["https://openalex.org/W2096733369","https://openalex.org/W2144012961","https://openalex.org/W2753840835","https://openalex.org/W4400615682","https://openalex.org/W4403713301"],"related_works":["https://openalex.org/W2628861693","https://openalex.org/W3203087560","https://openalex.org/W2099421762","https://openalex.org/W4361279463","https://openalex.org/W2530546662","https://openalex.org/W2967030268","https://openalex.org/W2132659060","https://openalex.org/W2031992971","https://openalex.org/W3214791684","https://openalex.org/W2152662039"],"abstract_inverted_index":{"The":[0,78,99,172],"Multimodal":[1,41],"Sentiment":[2],"Analysis":[3],"Challenge":[4],"presents":[5],"two":[6],"distinct":[7],"sub-challenges":[8],"related":[9],"to":[10,23,46,142,200],"human":[11,205],"perception":[12,206],"characteristics.":[13],"This":[14],"paper":[15],"focuses":[16],"on":[17],"the":[18,25,58,68,116,120,128,133,176,182,191,196,202],"MUSE-PERCEPTION":[19],"challenge,":[20],"which":[21],"aims":[22],"predict":[24,143],"perceptual":[26,145],"attributes":[27],"of":[28,61,70,119,132,178,198,204],"CEOs":[29],"from":[30,73],"video":[31],"data.":[32],"Specifically,":[33],"we":[34,151],"propose":[35],"a":[36,74,103,159,186],"novel":[37],"approach":[38],"that":[39],"leverages":[40],"Large":[42],"Language":[43],"Models":[44],"(MM-LLM)":[45],"integrate":[47],"multimodal":[48,81],"features":[49,92],"and":[50,67,90,106,122,157,165,181],"generate":[51],"accurate":[52],"predictions.":[53],"Our":[54],"proposal":[55],"explores":[56],"both":[57],"direct":[59],"generation":[60],"target":[62],"values":[63],"using":[64],"tuned":[65],"Qwen-VL":[66],"extraction":[69],"semantic":[71,117],"representations":[72,141],"pre-trained":[75],"Gemma-2B":[76],"model.":[77],"latter":[79],"integrates":[80],"data":[82,101],"such":[83,93],"as":[84,94],"facial":[85],"expressions":[86],"(vit-fer),":[87],"audio":[88,108],"(w2v-msp)":[89],"additional":[91],"egemaps":[95],"or":[96,124],"facenet512":[97],"results.":[98],"input":[100],"involve":[102],"task-specific":[104],"cue":[105],"an":[107,136],"transcript.":[109],"Feature":[110],"projectors":[111],"map":[112],"each":[113],"modality":[114],"into":[115],"space":[118],"LLM,":[121],"mean":[123,161],"attention":[125],"clustering":[126],"aggregates":[127],"final":[129],"hidden":[130],"state":[131],"LLM.":[134],"Next,":[135],"MLP":[137],"uses":[138],"these":[139],"aggregated":[140],"16":[144],"attributes.":[146],"To":[147],"optimize":[148],"model":[149,184],"performance,":[150],"experimented":[152],"with":[153],"several":[154],"fitting":[155],"methods":[156],"implemented":[158],"combined":[160],"absolute":[162],"error":[163],"(MAE)":[164],"Pearson":[166],"correlation":[167],"coefficient":[168],"(\u03c1)":[169],"loss":[170],"function.":[171],"experimental":[173],"results":[174,194],"shows":[175],"effectiveness":[177],"this":[179],"approach,":[180],"Gemma-2B-based":[183],"achieves":[185],"solid":[187],"second":[188],"place":[189],"in":[190,207],"challenge.":[192],"These":[193],"underscore":[195],"potential":[197],"MM-LLMs":[199],"improve":[201],"understanding":[203],"professional":[208],"contexts.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2025-12-27T23:08:20.325037","created_date":"2025-10-10T00:00:00"}