{"id":"https://openalex.org/W4408145566","doi":"https://doi.org/10.1109/tmm.2025.3535292","title":"Emotion-Oriented Cross-Modal Prompting and Alignment for Human-Centric Emotional Video Captioning","display_name":"Emotion-Oriented Cross-Modal Prompting and Alignment for Human-Centric Emotional Video Captioning","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4408145566","doi":"https://doi.org/10.1109/tmm.2025.3535292"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2025.3535292","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2025.3535292","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100445118","display_name":"Yu Wang","orcid":"https://orcid.org/0000-0001-8873-4778"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Wang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China","School of Computer Science and The School of Geography and Information Engineering, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0001-8873-4778","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]},{"raw_affiliation_string":"School of Computer Science and The School of Geography and Information Engineering, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100753233","display_name":"Yuanyuan Liu","orcid":"https://orcid.org/0000-0003-0465-3976"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanyuan Liu","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0003-0465-3976","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021637713","display_name":"Shunping Zhou","orcid":"https://orcid.org/0000-0002-2697-3383"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shunping Zhou","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-2697-3383","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106404215","display_name":"Yuxuan Huang","orcid":"https://orcid.org/0009-0007-1094-934X"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxuan Huang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5033240507","display_name":"Chang Tang","orcid":"https://orcid.org/0000-0002-6515-7696"},"institutions":[{"id":"https://openalex.org/I3124059619","display_name":"China University of Geosciences","ror":"https://ror.org/04gcegc37","country_code":"CN","type":"education","lineage":["https://openalex.org/I3124059619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chang Tang","raw_affiliation_strings":["School of Computer Science, China University of Geosciences, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-6515-7696","affiliations":[{"raw_affiliation_string":"School of Computer Science, China University of Geosciences, Wuhan, China","institution_ids":["https://openalex.org/I3124059619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076872319","display_name":"Wujie Zhou","orcid":"https://orcid.org/0000-0002-3055-2493"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wujie Zhou","raw_affiliation_strings":["School of Information and Electronic Engineering, Zhejiang University of Science and Technology, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-3055-2493","affiliations":[{"raw_affiliation_string":"School of Information and Electronic Engineering, Zhejiang University of Science and Technology, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100457637","display_name":"Zhe Chen","orcid":"https://orcid.org/0000-0001-5004-8975"},"institutions":[{"id":"https://openalex.org/I196829312","display_name":"La Trobe University","ror":"https://ror.org/01rxfrp27","country_code":"AU","type":"education","lineage":["https://openalex.org/I196829312"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Zhe Chen","raw_affiliation_strings":["Cisco-La Trobe Centre for AI and IoT, School of Computing, Engineering and Mathematical Sciences, La Trobe University, Bundoora, VIC, Australia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cisco-La Trobe Centre for AI and IoT, School of Computing, Engineering and Mathematical Sciences, La Trobe University, Bundoora, VIC, Australia","institution_ids":["https://openalex.org/I196829312"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":5.8242,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":{"value":0.96588869,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"27","issue":null,"first_page":"3766","last_page":"3780"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.904261589050293},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8600999116897583},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.5967975854873657},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43983498215675354},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.418865442276001},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.4127557575702667},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.35948777198791504},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3263590931892395},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.07985898852348328}],"concepts":[{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.904261589050293},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8600999116897583},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.5967975854873657},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43983498215675354},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.418865442276001},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.4127557575702667},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.35948777198791504},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3263590931892395},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.07985898852348328},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2025.3535292","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2025.3535292","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.41999998688697815,"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality"}],"awards":[{"id":"https://openalex.org/G1034316256","display_name":null,"funder_award_id":"2023AFB572","funder_id":"https://openalex.org/F4320322186","funder_display_name":"Natural Science Foundation of Hubei Province"},{"id":"https://openalex.org/G3057168972","display_name":null,"funder_award_id":"KLIGIP-2022-B10","funder_id":"https://openalex.org/F8678731271","funder_display_name":"Hubei Key Laboratory of Intelligent Geo-Information Processing"},{"id":"https://openalex.org/G4271941372","display_name":null,"funder_award_id":"62076227","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322186","display_name":"Natural Science Foundation of Hubei Province","ror":null},{"id":"https://openalex.org/F8678731271","display_name":"Hubei Key Laboratory of Intelligent Geo-Information Processing","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W1586939924","https://openalex.org/W1956340063","https://openalex.org/W2138621090","https://openalex.org/W2194775991","https://openalex.org/W2222512263","https://openalex.org/W2425121537","https://openalex.org/W2798833386","https://openalex.org/W2896457183","https://openalex.org/W2921644959","https://openalex.org/W2948358897","https://openalex.org/W2951390634","https://openalex.org/W2962681491","https://openalex.org/W2963430933","https://openalex.org/W2984862483","https://openalex.org/W2988753485","https://openalex.org/W3018349615","https://openalex.org/W3024308166","https://openalex.org/W3034221024","https://openalex.org/W3081168214","https://openalex.org/W3094502228","https://openalex.org/W3131477521","https://openalex.org/W3163971663","https://openalex.org/W3166366124","https://openalex.org/W3176689360","https://openalex.org/W3193767255","https://openalex.org/W3216659302","https://openalex.org/W3217340782","https://openalex.org/W4205803884","https://openalex.org/W4212830455","https://openalex.org/W4285200050","https://openalex.org/W4285601347","https://openalex.org/W4304099179","https://openalex.org/W4312056919","https://openalex.org/W4312463400","https://openalex.org/W4312560592","https://openalex.org/W4312877428","https://openalex.org/W4312941859","https://openalex.org/W4318953563","https://openalex.org/W4361224497","https://openalex.org/W4382240304","https://openalex.org/W4385768038","https://openalex.org/W4386076140","https://openalex.org/W4387969035","https://openalex.org/W4389776363","https://openalex.org/W4389903841","https://openalex.org/W4390874542","https://openalex.org/W4391454521","https://openalex.org/W4392947711","https://openalex.org/W4393156115","https://openalex.org/W4403791015","https://openalex.org/W6639432524","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6684090549","https://openalex.org/W6766978945","https://openalex.org/W6769627184","https://openalex.org/W6791353385","https://openalex.org/W6809975043"],"related_works":["https://openalex.org/W4210416330","https://openalex.org/W2775506363","https://openalex.org/W3088136942","https://openalex.org/W2963177403","https://openalex.org/W4290852288","https://openalex.org/W2949362007","https://openalex.org/W4283207562","https://openalex.org/W2330246314","https://openalex.org/W2949522393","https://openalex.org/W4289422896"],"abstract_inverted_index":{"Human-centric":[0],"Emotional":[1],"Video":[2],"Captioning":[3],"(H-EVC)":[4],"aims":[5],"to":[6,107,130,163],"generate":[7],"fine-grained,":[8],"emotion-related":[9],"sentences":[10],"for":[11],"human-based":[12],"videos,":[13],"enhancing":[14],"the":[15,42,149,153,160,176],"understanding":[16],"of":[17,97,143,178,199],"human":[18],"emotions":[19],"and":[20,35,80,103,112,126,133,152,165,169,206,217,223,228],"facilitating":[21],"human-computer":[22],"emotional":[23,33,47,115,124,180],"interaction.":[24],"However,":[25],"existing":[26],"video":[27,184],"captioning":[28],"methods":[29,192],"often":[30],"overlook":[31],"subtle":[32],"clues":[34],"interactions":[36],"in":[37,183],"videos.":[38],"As":[39],"a":[40],"result,":[41],"generated":[43],"captions":[44],"frequently":[45],"lack":[46],"information.":[48],"To":[49],"address":[50],"this,":[51],"we":[52],"proposeEmotion-orientedCross-modalPrompting":[53],"andAlignment":[54],"(ECPA),":[55],"which":[56],"improves":[57],"HEVC":[58],"accuracy":[59],"by":[60],"modeling":[61],"fine-grained":[62,179],"visual-textual":[63],"emotion":[64,77,82,100,136,172],"clues.":[65],"Using":[66],"large":[67],"foundation":[68],"models,":[69],"ECPA":[70,188],"introduces":[71,139],"two":[72,95,141],"learnable":[73,120],"prompting":[74,78,83],"strategies:":[75],"visual":[76,98,114],"(VEP)":[79],"textual":[81,121,135],"(TEP),":[84],"along":[85],"with":[86],"an":[87],"emotion-oriented":[88,144],"cross-modal":[89,171],"alignment":[90,146,156],"(ECA)":[91],"module.":[92],"VEP":[93],"uses":[94],"levels":[96,142],"prompts,i.e.,":[99,122],"recognition":[101],"(ER)":[102],"action":[104],"unit":[105],"(AU),":[106],"focus":[108],"on":[109,193,204,211,221],"both":[110,167],"coarse":[111],"fine":[113],"features.":[116],"TEP":[117],"devise":[118],"two-level":[119],"sentence-level":[123],"tokens":[125,129],"word-level":[127],"masked":[128],"capture":[131,164],"global":[132,168],"local":[134,170],"representations.":[137],"ECA":[138],"another":[140],"prompt":[145],"learning":[147],"mechanisms:":[148],"ER-sentence":[150],"level":[151,155],"AU-word":[154],"losses.":[157],"Both":[158],"enhance":[159],"model's":[161],"ability":[162],"integrate":[166],"semantics,":[173],"thereby":[174],"enabling":[175],"generation":[177],"linguistic":[181],"descriptions":[182],"captioning.":[185],"Experiments":[186],"show":[187],"significantly":[189],"outperforms":[190],"state-of-the-art":[191],"various":[194],"H-EVC":[195],"datasets":[196],"(relative":[197],"improvements":[198],"9.98%,":[200],"5.72%,":[201],"4.46%,":[202],"24.52%":[203],"MAFW,":[205],"12.82%,":[207],"20.27%,":[208],"4.22%,":[209],"5.01%":[210],"EmVidCap":[212],"across":[213],"four":[214],"evaluation":[215],"metrics)":[216],"supports":[218],"zero-shot":[219],"tasks":[220],"MSVD":[222],"MSRVTT,":[224],"demonstrating":[225],"strong":[226],"applicability":[227],"generalization.":[229]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":5}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-10-10T00:00:00"}