{"id":"https://openalex.org/W3207005999","doi":"https://doi.org/10.1145/3474085.3475638","title":"Hybrid Reasoning Network for Video-based Commonsense Captioning","display_name":"Hybrid Reasoning Network for Video-based Commonsense Captioning","publication_year":2021,"publication_date":"2021-10-17","ids":{"openalex":"https://openalex.org/W3207005999","doi":"https://doi.org/10.1145/3474085.3475638","mag":"3207005999"},"language":"en","primary_location":{"id":"doi:10.1145/3474085.3475638","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3474085.3475638","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Multimedia","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055989750","display_name":"Weijiang Yu","orcid":"https://orcid.org/0000-0002-7449-3093"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weijiang Yu","raw_affiliation_strings":["Sun Yat-sen University, Guangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056287596","display_name":"Jian Liang","orcid":"https://orcid.org/0000-0001-6298-7718"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Liang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100754485","display_name":"Lei Ji","orcid":"https://orcid.org/0000-0002-7569-3265"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Ji","raw_affiliation_strings":["Microsoft Research Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100658678","display_name":"Li L\u00fc","orcid":"https://orcid.org/0000-0001-5230-3749"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu Li","raw_affiliation_strings":["Zhejiang University, Hangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088688674","display_name":"Yuejian Fang","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuejian Fang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023506057","display_name":"Nong Xiao","orcid":"https://orcid.org/0000-0002-2166-977X"},"institutions":[{"id":"https://openalex.org/I157773358","display_name":"Sun Yat-sen University","ror":"https://ror.org/0064kty71","country_code":"CN","type":"education","lineage":["https://openalex.org/I157773358"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nong Xiao","raw_affiliation_strings":["Sun Yat-sen University, Guangzhou, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Sun Yat-sen University, Guangzhou, China","institution_ids":["https://openalex.org/I157773358"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042018181","display_name":"Nan Duan","orcid":"https://orcid.org/0000-0002-3387-4674"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Nan Duan","raw_affiliation_strings":["Microsoft Research Asia, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research Asia, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"5213","last_page":"5221"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9902999997138977,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"Commonsense reasoning","score":0.8894909620285034},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8363201022148132},{"id":"https://openalex.org/keywords/commonsense-knowledge","display_name":"Commonsense knowledge","score":0.8235296010971069},{"id":"https://openalex.org/keywords/closed-captioning","display_name":"Closed captioning","score":0.7974249720573425},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5701967477798462},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.5150875449180603},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4598584473133087},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.4155201315879822},{"id":"https://openalex.org/keywords/knowledge-representation-and-reasoning","display_name":"Knowledge representation and reasoning","score":0.20525622367858887},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.07448408007621765}],"concepts":[{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.8894909620285034},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8363201022148132},{"id":"https://openalex.org/C30542707","wikidata":"https://www.wikidata.org/wiki/Q1603203","display_name":"Commonsense knowledge","level":3,"score":0.8235296010971069},{"id":"https://openalex.org/C157657479","wikidata":"https://www.wikidata.org/wiki/Q2367247","display_name":"Closed captioning","level":3,"score":0.7974249720573425},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5701967477798462},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.5150875449180603},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4598584473133087},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4155201315879822},{"id":"https://openalex.org/C161301231","wikidata":"https://www.wikidata.org/wiki/Q3478658","display_name":"Knowledge representation and reasoning","level":2,"score":0.20525622367858887},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.07448408007621765},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3474085.3475638","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3474085.3475638","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 29th ACM International Conference on Multimedia","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1178782059","display_name":null,"funder_award_id":"NSFCU1811461","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2139608131","display_name":null,"funder_award_id":"2018B030312002","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321921","display_name":"Natural Science Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W46519926","https://openalex.org/W1566289585","https://openalex.org/W1596841185","https://openalex.org/W1773149199","https://openalex.org/W1933349210","https://openalex.org/W1956340063","https://openalex.org/W1995820507","https://openalex.org/W2064675550","https://openalex.org/W2078238240","https://openalex.org/W2101105183","https://openalex.org/W2111078031","https://openalex.org/W2123301721","https://openalex.org/W2139117248","https://openalex.org/W2139501017","https://openalex.org/W2164290393","https://openalex.org/W2194775991","https://openalex.org/W2342350679","https://openalex.org/W2423576022","https://openalex.org/W2425121537","https://openalex.org/W2463873473","https://openalex.org/W2463955103","https://openalex.org/W2544224704","https://openalex.org/W2600463316","https://openalex.org/W2739107216","https://openalex.org/W2748195089","https://openalex.org/W2768661419","https://openalex.org/W2787052186","https://openalex.org/W2795840542","https://openalex.org/W2892181857","https://openalex.org/W2898867267","https://openalex.org/W2949380904","https://openalex.org/W2949433733","https://openalex.org/W2950019618","https://openalex.org/W2950761309","https://openalex.org/W2963115613","https://openalex.org/W2963293463","https://openalex.org/W2963346996","https://openalex.org/W2963351113","https://openalex.org/W2963403868","https://openalex.org/W2963524571","https://openalex.org/W2963843052","https://openalex.org/W2963976294","https://openalex.org/W2963995027","https://openalex.org/W2969862959","https://openalex.org/W2970062726","https://openalex.org/W2970231061","https://openalex.org/W2970641692","https://openalex.org/W2995643077","https://openalex.org/W3035399403","https://openalex.org/W3035651653","https://openalex.org/W3101767943","https://openalex.org/W4249013746","https://openalex.org/W4297909472"],"related_works":["https://openalex.org/W3035583586","https://openalex.org/W4320165839","https://openalex.org/W2151799802","https://openalex.org/W4385488510","https://openalex.org/W2196562041","https://openalex.org/W2073302931","https://openalex.org/W4378501473","https://openalex.org/W3217195652","https://openalex.org/W3206107299","https://openalex.org/W3082691151"],"abstract_inverted_index":{"The":[0],"task":[1],"of":[2,51,72,103,173],"video-based":[3],"commonsense":[4,15,33,41,90,105,177],"captioning":[5],"aims":[6],"to":[7,64,116,169],"generate":[8],"event-wise":[9,107],"captions":[10,34,108],"and":[11,20,46,75,109,180],"meanwhile":[12],"provide":[13],"multiple":[14,104],"descriptions":[16],"(e.g.,":[17],"attribute,":[18],"effect":[19],"intention)":[21],"about":[22],"the":[23,27,32,49,66,70,98,101,118,126,131,143,149,153,157,164,184],"underlying":[24],"event":[25],"in":[26,92],"video.":[28],"Prior":[29],"works":[30],"explore":[31],"by":[35,86,147],"using":[36],"separate":[37],"networks":[38,68],"for":[39,83,160,176],"different":[40,52,89],"types,":[42],"which":[43,96],"is":[44],"time-consuming":[45],"lacks":[47],"mining":[48],"interaction":[50,99],"commonsense.":[53],"In":[54],"this":[55],"paper,":[56],"we":[57,79],"propose":[58],"a":[59,93,122,136],"Hybrid":[60],"Reasoning":[61],"Network":[62],"(HybridNet)":[63],"endow":[65],"neural":[67],"with":[69,196],"capability":[71],"semantic-level":[73,84],"reasoning":[74,85],"word-level":[76,119,144,161],"reasoning.":[77,162,178],"Firstly,":[78],"develop":[80],"multi-commonsense":[81],"learning":[82],"jointly":[87],"training":[88],"types":[91],"unified":[94],"network,":[95],"encourages":[97],"between":[100],"clues":[102],"descriptions,":[106],"videos.":[110],"Then,":[111],"there":[112],"are":[113,167],"two":[114],"steps":[115],"achieve":[117],"reasoning:":[120],"(1)":[121],"memory":[123,154],"module":[124,141,155],"records":[125],"history":[127,150],"predicted":[128],"sequence":[129],"from":[130,152],"previous":[132],"generation":[133],"processes;":[134],"(2)":[135],"memory-routed":[137],"multi-head":[138],"attention":[139,145],"(MMHA)":[140],"updates":[142],"maps":[146],"incorporating":[148],"information":[151],"into":[156],"transformer":[158],"decoder":[159],"Moreover,":[163],"multimodal":[165],"features":[166],"used":[168],"make":[170],"full":[171],"use":[172],"diverse":[174],"knowledge":[175],"Experiments":[179],"abundant":[181],"analysis":[182],"on":[183],"large-scale":[185],"Video-to-Commonsense":[186],"benchmark":[187],"show":[188],"that":[189],"our":[190],"HybridNet":[191],"achieves":[192],"state-of-the-art":[193],"performance":[194],"compared":[195],"other":[197],"methods.":[198]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}