{"id":"https://openalex.org/W7124435881","doi":"https://doi.org/10.1109/tmm.2026.3654377","title":"Language-Guided Multimodal Spiking Neural Networks for Event-Based Action Recognition","display_name":"Language-Guided Multimodal Spiking Neural Networks for Event-Based Action Recognition","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7124435881","doi":"https://doi.org/10.1109/tmm.2026.3654377"},"language":null,"primary_location":{"id":"doi:10.1109/tmm.2026.3654377","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2026.3654377","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026088596","display_name":"Ziliang Ren","orcid":"https://orcid.org/0000-0001-7940-294X"},"institutions":[{"id":"https://openalex.org/I2799850029","display_name":"Dongguan University of Technology","ror":"https://ror.org/01m8p7q42","country_code":"CN","type":"education","lineage":["https://openalex.org/I2799850029"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Ziliang Ren","raw_affiliation_strings":["School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China"],"raw_orcid":"https://orcid.org/0000-0001-7940-294X","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China","institution_ids":["https://openalex.org/I2799850029"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123207525","display_name":"Jiaqi Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I2799850029","display_name":"Dongguan University of Technology","ror":"https://ror.org/01m8p7q42","country_code":"CN","type":"education","lineage":["https://openalex.org/I2799850029"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaqi Chen","raw_affiliation_strings":["School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China"],"raw_orcid":"https://orcid.org/0009-0000-5139-6488","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Dongguan University of Technology, Dongguan, China","institution_ids":["https://openalex.org/I2799850029"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054902681","display_name":"Fuxiang Wu","orcid":"https://orcid.org/0000-0002-4542-4486"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fuxiang Wu","raw_affiliation_strings":["CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-4542-4486","affiliations":[{"raw_affiliation_string":"CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China","institution_ids":["https://openalex.org/I4210145761","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qieshi Zhang","orcid":"https://orcid.org/0000-0001-6358-1840"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qieshi Zhang","raw_affiliation_strings":["CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0001-6358-1840","affiliations":[{"raw_affiliation_string":"CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China","institution_ids":["https://openalex.org/I4210145761","https://openalex.org/I19820366"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5123202423","display_name":"Jun Cheng","orcid":null},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210145761","display_name":"Shenzhen Institutes of Advanced Technology","ror":"https://ror.org/04gh4er46","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210145761"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Cheng","raw_affiliation_strings":["CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China"],"raw_orcid":"https://orcid.org/0000-0002-3131-3275","affiliations":[{"raw_affiliation_string":"CAS Key Laboratory of Human-Machine Intelligence-Synergy Systems, Shenzhen Institutes of Advanced Technology, Chinese Academy of Sciences, Shenzhen, China","institution_ids":["https://openalex.org/I4210145761","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5026088596"],"corresponding_institution_ids":["https://openalex.org/I2799850029"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.07465898,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"28","issue":null,"first_page":"3667","last_page":"3681"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.7936999797821045,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.7936999797821045,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11431","display_name":"Action Observation and Synchronization","score":0.0421999990940094,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.03819999843835831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.6489999890327454},{"id":"https://openalex.org/keywords/action-recognition","display_name":"Action recognition","score":0.6488000154495239},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5156999826431274},{"id":"https://openalex.org/keywords/event","display_name":"Event (particle physics)","score":0.4846999943256378},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.4569000005722046},{"id":"https://openalex.org/keywords/feature-extraction","display_name":"Feature extraction","score":0.4255000054836273},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.42100000381469727},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.4147000014781952},{"id":"https://openalex.org/keywords/neuromorphic-engineering","display_name":"Neuromorphic engineering","score":0.37209999561309814}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8769999742507935},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.664900004863739},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.6489999890327454},{"id":"https://openalex.org/C2987834672","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Action recognition","level":3,"score":0.6488000154495239},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5156999826431274},{"id":"https://openalex.org/C2779662365","wikidata":"https://www.wikidata.org/wiki/Q5416694","display_name":"Event (particle physics)","level":2,"score":0.4846999943256378},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.4569000005722046},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43950000405311584},{"id":"https://openalex.org/C52622490","wikidata":"https://www.wikidata.org/wiki/Q1026626","display_name":"Feature extraction","level":2,"score":0.4255000054836273},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.42100000381469727},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.4147000014781952},{"id":"https://openalex.org/C151927369","wikidata":"https://www.wikidata.org/wiki/Q1981312","display_name":"Neuromorphic engineering","level":3,"score":0.37209999561309814},{"id":"https://openalex.org/C97931131","wikidata":"https://www.wikidata.org/wiki/Q5282087","display_name":"Discriminative model","level":2,"score":0.35409998893737793},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.35350000858306885},{"id":"https://openalex.org/C11731999","wikidata":"https://www.wikidata.org/wiki/Q9067355","display_name":"Spiking neural network","level":3,"score":0.33640000224113464},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3353999853134155},{"id":"https://openalex.org/C2781122975","wikidata":"https://www.wikidata.org/wiki/Q16928266","display_name":"Semantic feature","level":2,"score":0.3310999870300293},{"id":"https://openalex.org/C64876066","wikidata":"https://www.wikidata.org/wiki/Q5141226","display_name":"Cognitive neuroscience of visual object recognition","level":3,"score":0.3255000114440918},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.3249000012874603},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.29980000853538513},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C133488467","wikidata":"https://www.wikidata.org/wiki/Q6673524","display_name":"Long short term memory","level":4,"score":0.2906999886035919},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.2824000120162964},{"id":"https://openalex.org/C121687571","wikidata":"https://www.wikidata.org/wiki/Q4677630","display_name":"Activity recognition","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.2671999931335449},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.2648000121116638},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2596000134944916},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.25920000672340393},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.25529998540878296}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tmm.2026.3654377","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tmm.2026.3654377","pdf_url":null,"source":{"id":"https://openalex.org/S137030581","display_name":"IEEE Transactions on Multimedia","issn_l":"1520-9210","issn":["1520-9210","1941-0077"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Multimedia","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","score":0.9027364253997803,"id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G253655306","display_name":null,"funder_award_id":"62376261","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5258257467","display_name":null,"funder_award_id":"U21A20487","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G587205070","display_name":null,"funder_award_id":"62372440","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7168745877","display_name":null,"funder_award_id":"2024A1515011754","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321921","display_name":"Natural Science Foundation of Guangdong Province","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Event-based":[0],"human":[1],"action":[2,19,77,161],"recognition":[3,20,57,162],"has":[4],"gained":[5],"increasing":[6],"attention":[7],"due":[8],"to":[9,34,75,95,106,137],"its":[10],"efficiency":[11,167],"in":[12,157],"dynamic":[13],"scenarios.":[14],"Contemporary":[15],"methodologies":[16],"for":[17],"event-based":[18,160],"predominantly":[21],"treat":[22],"the":[23,36,97,103,140,146,152,165],"problem":[24],"as":[25,80],"a":[26,48,66,89],"one-hot":[27],"classification":[28],"task,":[29],"which":[30,55],"limits":[31],"their":[32],"ability":[33],"leverage":[35],"semantic":[37,81],"relationships":[38],"among":[39],"various":[40],"actions.":[41,111],"To":[42],"address":[43],"this":[44],"limitation,":[45],"we":[46],"propose":[47],"Spiking":[49],"Event-Text":[50],"Feature":[51],"Fusion":[52],"(SETFF)":[53],"framework,":[54],"enhances":[56],"performance":[58],"by":[59],"integrating":[60],"event":[61,85],"and":[62,109,123,143],"text":[63],"modalities":[64],"through":[65],"dual-stream":[67],"architecture.":[68],"SETFF":[69,127],"leverages":[70],"generative":[71],"large":[72],"language":[73],"models":[74],"produce":[76],"descriptions,":[78],"serving":[79],"prompts":[82],"that":[83,126],"guide":[84],"feature":[86],"learning.":[87],"Specifically,":[88],"contrastive":[90],"loss":[91],"function":[92],"is":[93],"employed":[94],"align":[96],"features":[98],"of":[99,135,154,169],"both":[100],"modalities,":[101],"enriching":[102],"model's":[104],"capacity":[105],"distinguish":[107],"intricate":[108],"subtle":[110],"Extensive":[112],"experiments":[113],"on":[114,139,145],"neuromorphic":[115],"datasets,":[116],"including":[117],"PAF,":[118],"DailyAction-DVS,":[119],"DVS128":[120],"Gesture,":[121],"Bullying10K,":[122],"UCF101-DVS,":[124],"demonstrate":[125],"achieves":[128],"state-of-the-art":[129],"accuracy,":[130],"with":[131],"top-1":[132],"accuracy":[133],"rates":[134],"up":[136],"99.65%":[138],"DailyAction-DVS":[141],"dataset":[142],"98.39%":[144],"PAF":[147],"dataset.":[148],"Experimental":[149],"results":[150],"underscore":[151],"effectiveness":[153],"multimodal":[155],"fusion":[156],"SNNs,":[158],"advancing":[159],"while":[163],"preserving":[164],"energy":[166],"characteristic":[168],"SNNs.":[170]},"counts_by_year":[],"updated_date":"2026-05-23T06:10:36.450269","created_date":"2026-01-17T00:00:00"}