{"id":"https://openalex.org/W4401414469","doi":"https://doi.org/10.1109/icra57147.2024.10611454","title":"Incorporating Scene Graphs into Pre-trained Vision-Language Models for Multimodal Open-vocabulary Action Recognition","display_name":"Incorporating Scene Graphs into Pre-trained Vision-Language Models for Multimodal Open-vocabulary Action Recognition","publication_year":2024,"publication_date":"2024-05-13","ids":{"openalex":"https://openalex.org/W4401414469","doi":"https://doi.org/10.1109/icra57147.2024.10611454"},"language":"en","primary_location":{"id":"doi:10.1109/icra57147.2024.10611454","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icra57147.2024.10611454","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100679851","display_name":"Chao Wei","orcid":"https://orcid.org/0000-0002-2278-7057"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chao Wei","raw_affiliation_strings":["Tsinghua University,BNRist, THUAI,Department of Computer Science,Beijing,China,100084"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist, THUAI,Department of Computer Science,Beijing,China,100084","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102011846","display_name":"Zhidong Deng","orcid":"https://orcid.org/0000-0001-9970-1023"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhidong Deng","raw_affiliation_strings":["Tsinghua University,BNRist, THUAI,Department of Computer Science,Beijing,China,100084"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University,BNRist, THUAI,Department of Computer Science,Beijing,China,100084","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"440","last_page":"447"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9603999853134155,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7786855101585388},{"id":"https://openalex.org/keywords/action-recognition","display_name":"Action recognition","score":0.7455543279647827},{"id":"https://openalex.org/keywords/vocabulary","display_name":"Vocabulary","score":0.6486932635307312},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6086949110031128},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5329810976982117},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5216977000236511},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4259078800678253},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.38301482796669006},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.15036270022392273}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7786855101585388},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.7455543279647827},{"id":"https://openalex.org/C2777601683","wikidata":"https://www.wikidata.org/wiki/Q6499736","display_name":"Vocabulary","level":2,"score":0.6486932635307312},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6086949110031128},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5329810976982117},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5216977000236511},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4259078800678253},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.38301482796669006},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15036270022392273},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icra57147.2024.10611454","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1109/icra57147.2024.10611454","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Robotics and Automation (ICRA)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6899999976158142}],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":91,"referenced_works":["https://openalex.org/W1522734439","https://openalex.org/W1527575280","https://openalex.org/W1927052826","https://openalex.org/W2077069816","https://openalex.org/W2123024445","https://openalex.org/W2149557440","https://openalex.org/W2277195237","https://openalex.org/W2337252826","https://openalex.org/W2507009361","https://openalex.org/W2559085405","https://openalex.org/W2579549467","https://openalex.org/W2603705233","https://openalex.org/W2619697695","https://openalex.org/W2619947201","https://openalex.org/W2774267535","https://openalex.org/W2798991696","https://openalex.org/W2807340089","https://openalex.org/W2921906393","https://openalex.org/W2944006115","https://openalex.org/W2962688385","https://openalex.org/W2962960500","https://openalex.org/W2963524571","https://openalex.org/W2963536419","https://openalex.org/W2963703197","https://openalex.org/W2963722382","https://openalex.org/W2964134613","https://openalex.org/W2970231061","https://openalex.org/W2972073579","https://openalex.org/W2984008963","https://openalex.org/W2990503944","https://openalex.org/W2994508843","https://openalex.org/W3010094231","https://openalex.org/W3034671389","https://openalex.org/W3034679267","https://openalex.org/W3035635319","https://openalex.org/W3037309139","https://openalex.org/W3090449556","https://openalex.org/W3095753865","https://openalex.org/W3097283600","https://openalex.org/W3106759358","https://openalex.org/W3108655343","https://openalex.org/W3129071613","https://openalex.org/W3160587007","https://openalex.org/W3175300676","https://openalex.org/W3175859344","https://openalex.org/W3176445421","https://openalex.org/W3195429019","https://openalex.org/W3204882374","https://openalex.org/W3209532394","https://openalex.org/W3213454282","https://openalex.org/W3215626407","https://openalex.org/W4200477908","https://openalex.org/W4225323055","https://openalex.org/W4229042118","https://openalex.org/W4282935537","https://openalex.org/W4287658071","https://openalex.org/W4287751139","https://openalex.org/W4297808394","https://openalex.org/W4306820534","https://openalex.org/W4309397398","https://openalex.org/W4312372834","https://openalex.org/W4312554543","https://openalex.org/W4312563197","https://openalex.org/W4312614039","https://openalex.org/W4312685069","https://openalex.org/W4312936847","https://openalex.org/W4313156423","https://openalex.org/W4313190371","https://openalex.org/W4386065512","https://openalex.org/W6631516269","https://openalex.org/W6638667902","https://openalex.org/W6678470764","https://openalex.org/W6747225742","https://openalex.org/W6752216966","https://openalex.org/W6763392795","https://openalex.org/W6775244068","https://openalex.org/W6780196292","https://openalex.org/W6780294235","https://openalex.org/W6783983736","https://openalex.org/W6790019176","https://openalex.org/W6791353385","https://openalex.org/W6802987763","https://openalex.org/W6803872405","https://openalex.org/W6810334672","https://openalex.org/W6811013733","https://openalex.org/W6811072154","https://openalex.org/W6844194202","https://openalex.org/W6845354634","https://openalex.org/W6846007759","https://openalex.org/W6846210491","https://openalex.org/W6955071965"],"related_works":["https://openalex.org/W2349784553","https://openalex.org/W3022596247","https://openalex.org/W2601444686","https://openalex.org/W4307058054","https://openalex.org/W4292238148","https://openalex.org/W4323660495","https://openalex.org/W2385319785","https://openalex.org/W2900827440","https://openalex.org/W1576128429","https://openalex.org/W2269464716"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"Action-SGFA,":[3],"a":[4,145],"novel":[5],"action":[6,17,51,73,101,151],"feature":[7],"alignment":[8],"approach":[9],"to":[10,33,87,98,120],"learn":[11],"unified":[12,90],"joint":[13],"embeddings":[14],"across":[15,154],"four":[16],"modalities":[18,102,118],"incorporating":[19],"scene":[20],"graph":[21,66],"(SG)":[22],"comprehension.":[23,142],"A":[24],"new":[25,117,146],"training":[26],"paradigm":[27],"for":[28,53],"Action-SGFA":[29,105],"is":[30,96],"also":[31],"devised":[32],"improve":[34],"pre-trained":[35,136],"VL":[36,110,137],"models":[37],"using":[38],"datasets":[39],"with":[40,124,131],"SG":[41,58,141],"annotation.":[42],"When":[43],"learning":[44],"from":[45],"image-SG":[46],"pairs,":[47],"it":[48],"captures":[49],"structure-associated":[50],"knowledge":[52],"visual":[54],"and":[55,92,139,165,169],"textual":[56],"encoders.":[57],"supervision":[59],"generates":[60],"fine-grained":[61],"captions":[62],"based":[63],"on":[64,167,184],"various":[65],"augmentations":[67],"highlighting":[68],"different":[69],"compositional":[70],"aspects":[71],"of":[72,82,116,134,176],"scenes.":[74],"Furthermore,":[75],"our":[76],"research":[77],"reveals":[78],"that":[79],"all":[80,100],"combinations":[81],"paired":[83],"data":[84,95],"are":[85],"unnecessary":[86],"train":[88],"such":[89],"embeddings,":[91],"only":[93],"image-paired":[94],"sufficient":[97],"bind":[99],"together.":[103],"Our":[104],"can":[106],"leverage":[107],"existing":[108],"large":[109],"models,":[111],"enhancing":[112],"their":[113,121],"zero-shot":[114,128,150,161],"capabilities":[115],"due":[119],"natural":[122],"pairings":[123],"images.":[125],"The":[126],"open-vocabulary":[127],"performance":[129],"improves":[130],"the":[132,135,140,158,174,181],"strength":[133],"model":[138],"We":[143],"establish":[144],"state-of-the-art":[147,182],"in":[148,173],"several":[149],"recognition":[152],"tasks":[153],"modalities,":[155],"significantly":[156],"surpassing":[157],"vanilla":[159],"skeleton":[160],"method":[162,183],"by":[163,186],"27.0%":[164],"19.7%":[166],"NTU-60":[168],"NTU-120,":[170],"respectively.":[171],"Additionally,":[172],"context":[175],"RGB":[177],"videos,":[178],"we":[179],"surpass":[180],"Kinetics-400":[185],"2.1%.":[187]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}