{"id":"https://openalex.org/W4377971516","doi":"https://doi.org/10.1109/tmm.2023.3279691","title":"Multi-Modal Structure-Embedding Graph Transformer for Visual Commonsense Reasoning","display_name":"Multi-Modal Structure-Embedding Graph Transformer for Visual Commonsense Reasoning","publication_year":2023,"publication_date":"2023-05-24","ids":{"openalex":"https://openalex.org/W4377971516","doi":"https://doi.org/10.1109/tmm.2023.3279691"},"language":"en","primary_location":{"id":"doi:10.1109/tmm.2023.3279691","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3279691","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101670573","display_name":"Jian Zhu","orcid":"https://orcid.org/0000-0002-3835-4627"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Zhu","raw_affiliation_strings":["Department of Computer Science and Technology, Key Laboratory of Embedded System and Service Computing (Ministry of Education), Tongji University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-3835-4627","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Key Laboratory of Embedded System and Service Computing (Ministry of Education), Tongji University, Shanghai, China","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058982350","display_name":"Hanli Wang","orcid":"https://orcid.org/0000-0002-9999-4871"},"institutions":[{"id":"https://openalex.org/I116953780","display_name":"Tongji University","ror":"https://ror.org/03rc6as71","country_code":"CN","type":"education","lineage":["https://openalex.org/I116953780"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanli Wang","raw_affiliation_strings":["Department of Computer Science and Technology, Key Laboratory of Embedded System and Service Computing (Ministry of Education), Tongji University, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0002-9999-4871","affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Key Laboratory of Embedded System and Service Computing (Ministry of Education), Tongji University, Shanghai, China","institution_ids":["https://openalex.org/I116953780"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5049557511","display_name":"Bin He","orcid":"https://orcid.org/0000-0003-3193-6269"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bin He","raw_affiliation_strings":["Frontiers Science Center for Intelligent Autonomous Systems, Shanghai, China"],"raw_orcid":"https://orcid.org/0000-0003-3193-6269","affiliations":[{"raw_affiliation_string":"Frontiers Science Center for Intelligent Autonomous Systems, Shanghai, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.0636,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.78824179,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"26","issue":null,"first_page":"1295","last_page":"1305"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10627","display_name":"Advanced Image and Video Retrieval Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11439","display_name":"Video Analysis and Summarization","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8047676086425781},{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"Commonsense reasoning","score":0.7272600531578064},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.675583004951477},{"id":"https://openalex.org/keywords/modal","display_name":"Modal","score":0.6330004930496216},{"id":"https://openalex.org/keywords/commonsense-knowledge","display_name":"Commonsense knowledge","score":0.6311401724815369},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5598242878913879},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.43936556577682495},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3985269069671631},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.27948999404907227},{"id":"https://openalex.org/keywords/knowledge-based-systems","display_name":"Knowledge-based systems","score":0.16532805562019348}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8047676086425781},{"id":"https://openalex.org/C193221554","wikidata":"https://www.wikidata.org/wiki/Q5153664","display_name":"Commonsense reasoning","level":2,"score":0.7272600531578064},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.675583004951477},{"id":"https://openalex.org/C71139939","wikidata":"https://www.wikidata.org/wiki/Q910194","display_name":"Modal","level":2,"score":0.6330004930496216},{"id":"https://openalex.org/C30542707","wikidata":"https://www.wikidata.org/wiki/Q1603203","display_name":"Commonsense knowledge","level":3,"score":0.6311401724815369},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5598242878913879},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.43936556577682495},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3985269069671631},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.27948999404907227},{"id":"https://openalex.org/C115925183","wikidata":"https://www.wikidata.org/wiki/Q1412694","display_name":"Knowledge-based systems","level":2,"score":0.16532805562019348},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C188027245","wikidata":"https://www.wikidata.org/wiki/Q750446","display_name":"Polymer chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2023.3279691","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2023.3279691","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5899999737739563}],"awards":[{"id":"https://openalex.org/G5501368935","display_name":null,"funder_award_id":"61976159","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5794821986","display_name":null,"funder_award_id":"62133011","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":46,"referenced_works":["https://openalex.org/W2194775991","https://openalex.org/W2471094925","https://openalex.org/W2608787653","https://openalex.org/W2745461083","https://openalex.org/W2896457183","https://openalex.org/W2963115613","https://openalex.org/W2963717374","https://openalex.org/W2964110616","https://openalex.org/W2968124245","https://openalex.org/W2996249958","https://openalex.org/W3007364240","https://openalex.org/W3010803012","https://openalex.org/W3018388102","https://openalex.org/W3021007069","https://openalex.org/W3023463084","https://openalex.org/W3026441479","https://openalex.org/W3034379414","https://openalex.org/W3035651653","https://openalex.org/W3090449556","https://openalex.org/W3092336341","https://openalex.org/W3096609285","https://openalex.org/W3099023595","https://openalex.org/W3131477521","https://openalex.org/W3160447289","https://openalex.org/W3174690404","https://openalex.org/W3176896951","https://openalex.org/W3177224328","https://openalex.org/W3185066916","https://openalex.org/W3194633557","https://openalex.org/W3209214366","https://openalex.org/W4284692930","https://openalex.org/W4304091902","https://openalex.org/W4383337770","https://openalex.org/W6631190155","https://openalex.org/W6728881024","https://openalex.org/W6739901393","https://openalex.org/W6767194493","https://openalex.org/W6767211374","https://openalex.org/W6767279747","https://openalex.org/W6767362881","https://openalex.org/W6769264886","https://openalex.org/W6779879114","https://openalex.org/W6789909235","https://openalex.org/W6791460281","https://openalex.org/W6796768152","https://openalex.org/W6802862198"],"related_works":["https://openalex.org/W3035583586","https://openalex.org/W4320165839","https://openalex.org/W2151799802","https://openalex.org/W4385488510","https://openalex.org/W2196562041","https://openalex.org/W2073302931","https://openalex.org/W3206107299","https://openalex.org/W3082691151","https://openalex.org/W4287633646","https://openalex.org/W4378501473"],"abstract_inverted_index":{"Visual":[0],"commonsense":[1],"reasoning":[2,7,222],"(VCR)":[3],"is":[4,60,116,167,214],"a":[5,19,25,61,107,163,210],"challenging":[6],"task":[8],"that":[9],"aims":[10],"to":[11,35,49,128,149,169,193,217],"not":[12],"only":[13,89],"answer":[14],"the":[15,29,39,95,99,175,180,229,232,241],"question":[16],"based":[17,54],"on":[18,55,240],"given":[20],"image":[21,42],"but":[22],"also":[23],"provide":[24],"rationale":[26],"justifying":[27],"for":[28,45,157,179,221],"choice.":[30],"Graph-based":[31],"networks":[32],"are":[33,126,144],"appropriate":[34],"represent":[36,129],"and":[37,43,51,71,94,122,130,133,146,153,185,197],"extract":[38,198],"correlation":[40,177],"between":[41],"language":[44],"reasoning,":[46],"where":[47,139],"how":[48],"construct":[50],"learn":[52],"graphs":[53],"such":[56],"multi-modal":[57,82,141],"Euclidean":[58],"data":[59],"fundamental":[62],"problem.":[63],"Most":[64],"existing":[65],"graph-based":[66],"methods":[67,239],"view":[68],"visual":[69,150],"regions":[70],"linguistic":[72,154],"words":[73],"as":[74,98],"identical":[75],"graph":[76,121,125,160,165,208],"nodes,":[77],"ignoring":[78],"inherent":[79],"characteristics":[80],"of":[81,182,231],"data.":[83],"In":[84],"addition,":[85],"these":[86,105],"approaches":[87],"typically":[88],"have":[90],"one":[91],"graph-learning":[92],"layer,":[93],"performance":[96],"declines":[97],"model":[100,131,195],"goes":[101],"deeper.":[102],"To":[103,205],"address":[104],"issues,":[106],"novel":[108],"method":[109],"named":[110],"Multi-modal":[111],"Structure-embedding":[112],"Graph":[113],"Transformer":[114],"(MSGT)":[115],"proposed.":[117],"Specifically,":[118],"an":[119,123],"answer-vision":[120],"answer-question":[124],"constructed":[127],"intra-modal":[132],"inter-modal":[134],"correlations":[135],"in":[136],"VCR":[137,242],"simultaneously,":[138],"additional":[140],"structure":[142,172,186],"representations":[143],"initialized":[145],"embedded":[147,171],"according":[148],"region":[151],"distances":[152],"word":[155],"orders":[156],"more":[158,191,199],"reasonable":[159],"representation.":[161],"Then,":[162],"structure-injecting":[164],"transformer":[166],"designed":[168],"inject":[170],"priors":[173],"into":[174],"semantic":[176],"matrix":[178],"evolution":[181],"node":[183,225],"features":[184,201],"representations,":[187],"which":[188],"can":[189],"stack":[190],"layers":[192],"make":[194],"deeper":[196],"powerful":[200],"with":[202,237],"instructive":[203],"priors.":[204],"adaptively":[206],"fuse":[207],"features,":[209],"scored":[211],"pooling":[212],"mechanism":[213],"further":[215],"developed":[216],"select":[218],"valuable":[219],"clues":[220],"from":[223],"learnt":[224],"features.":[226],"Experiments":[227],"demonstrate":[228],"superiority":[230],"proposed":[233],"MSGT":[234],"framework":[235],"compared":[236],"state-of-the-art":[238],"benchmark":[243],"dataset.":[244]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":5},{"year":2023,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}