{"id":"https://openalex.org/W4403791015","doi":"https://doi.org/10.1145/3664647.3681601","title":"Observe before Generate: Emotion-Cause aware Video Caption for Multimodal Emotion Cause Generation in Conversations","display_name":"Observe before Generate: Emotion-Cause aware Video Caption for Multimodal Emotion Cause Generation in Conversations","publication_year":2024,"publication_date":"2024-10-26","ids":{"openalex":"https://openalex.org/W4403791015","doi":"https://doi.org/10.1145/3664647.3681601"},"language":"en","primary_location":{"id":"doi:10.1145/3664647.3681601","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681601","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043764949","display_name":"Fanfan Wang","orcid":"https://orcid.org/0000-0002-8955-9411"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fanfan Wang","raw_affiliation_strings":["School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-8955-9411","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100547236","display_name":"Heqing Ma","orcid":"https://orcid.org/0009-0000-8830-193X"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Heqing Ma","raw_affiliation_strings":["School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0000-8830-193X","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028086891","display_name":"Xiangqing Shen","orcid":"https://orcid.org/0000-0002-9825-7853"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiangqing Shen","raw_affiliation_strings":["School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-9825-7853","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084990301","display_name":"Jianfei Yu","orcid":"https://orcid.org/0000-0001-8380-0609"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianfei Yu","raw_affiliation_strings":["School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-8380-0609","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035534911","display_name":"Rui Xia","orcid":"https://orcid.org/0000-0002-0621-1058"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Rui Xia","raw_affiliation_strings":["School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-0621-1058","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I36399199"],"apc_list":null,"apc_paid":null,"fwci":1.5776,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.88188219,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"5820","last_page":"5828"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.715545117855072},{"id":"https://openalex.org/keywords/emotion-recognition","display_name":"Emotion recognition","score":0.5450194478034973},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.43542882800102234},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3997132480144501},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3725014328956604}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.715545117855072},{"id":"https://openalex.org/C2777438025","wikidata":"https://www.wikidata.org/wiki/Q1339090","display_name":"Emotion recognition","level":2,"score":0.5450194478034973},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.43542882800102234},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3997132480144501},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3725014328956604}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3664647.3681601","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3664647.3681601","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 32nd ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Gender equality","score":0.4000000059604645,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":28,"referenced_works":["https://openalex.org/W799573885","https://openalex.org/W1593271688","https://openalex.org/W1956340063","https://openalex.org/W1966797434","https://openalex.org/W1993858167","https://openalex.org/W2182096631","https://openalex.org/W2538796508","https://openalex.org/W2562539671","https://openalex.org/W2600463316","https://openalex.org/W2740887992","https://openalex.org/W2766095568","https://openalex.org/W2951124019","https://openalex.org/W2963711448","https://openalex.org/W2981852735","https://openalex.org/W3153622894","https://openalex.org/W3199412388","https://openalex.org/W4285292976","https://openalex.org/W4285605578","https://openalex.org/W4288089799","https://openalex.org/W4307079201","https://openalex.org/W4312458895","https://openalex.org/W4312773554","https://openalex.org/W4360789483","https://openalex.org/W4385573523","https://openalex.org/W4387448596","https://openalex.org/W4389520248","https://openalex.org/W4393160270","https://openalex.org/W4401043674"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W3126677997","https://openalex.org/W1610857240"],"abstract_inverted_index":{"Emotion":[0,69],"cause":[1,29,172],"analysis":[2],"has":[3],"attracted":[4],"increasing":[5],"attention":[6],"in":[7,50,72],"recent":[8],"years.":[9],"However,":[10],"the":[11,37,86,90,95,135,159,165],"integration":[12],"of":[13,55,98,137,161,167],"multimodal":[14,96,168],"information":[15,169],"with":[16,146],"emotion":[17,92,113,138,171],"causes":[18,39,87,139],"remains":[19],"underexplored.":[20],"Existing":[21],"studies":[22],"merely":[23],"extract":[24],"utterances":[25],"from":[26,40,115],"conversations":[27,110],"as":[28],"evidence,":[30],"which":[31,75],"is":[32,144],"too":[33],"coarse-grained":[34],"to":[35,77],"locate":[36],"exact":[38],"other":[41],"modalities,":[42],"especially":[43],"those":[44],"that":[45,88,107,124],"may":[46],"be":[47],"reflected":[48],"only":[49],"a":[51,64,103,121,150],"specific":[52],"video":[53,129],"frame":[54],"an":[56,79],"utterance.":[57],"To":[58],"address":[59],"these":[60],"limitations,":[61],"we":[62],"introduce":[63],"new":[65],"task":[66],"named":[67,105],"Multimodal":[68,151],"Cause":[70],"Generation":[71],"Conversations":[73],"(MECGC),":[74],"aims":[76],"generate":[78],"abstractive":[80],"summary":[81],"clearly":[82],"and":[83,111,132,164,176],"intuitively":[84],"describing":[85],"trigger":[89],"given":[91],"based":[93],"on":[94],"context":[97],"conversations.":[99],"We":[100,118],"accordingly":[101],"construct":[102],"dataset":[104,175],"ECGF":[106],"contains":[108],"1,374":[109],"7,690":[112],"instances":[114],"TV":[116],"series.":[117],"further":[119],"develop":[120],"generative":[122],"framework":[123,163],"first":[125],"generates":[126],"emotion-cause":[127],"aware":[128],"captions":[130],"(Observe)":[131],"then":[133],"facilitates":[134],"generation":[136],"(Generate).":[140],"The":[141],"captioning":[142],"model":[143],"trained":[145],"examples":[147],"synthesized":[148],"by":[149],"Large":[152],"Language":[153],"Model":[154],"(MLLM).":[155],"Experimental":[156],"results":[157],"demonstrate":[158],"effectiveness":[160],"our":[162],"significance":[166],"for":[170],"analysis.":[173],"Our":[174],"source":[177],"codes":[178],"are":[179],"available":[180],"at":[181],"https://github.com/NUSTM/MECGC.":[182]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}