{"id":"https://openalex.org/W4405180690","doi":"https://doi.org/10.1109/tpami.2024.3511936","title":"Spiking Variational Policy Gradient for Brain Inspired Reinforcement Learning","display_name":"Spiking Variational Policy Gradient for Brain Inspired Reinforcement Learning","publication_year":2024,"publication_date":"2024-12-09","ids":{"openalex":"https://openalex.org/W4405180690","doi":"https://doi.org/10.1109/tpami.2024.3511936","pmid":"https://pubmed.ncbi.nlm.nih.gov/40030447"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2024.3511936","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3511936","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure-oai.bham.ac.uk/ws/files/252277110/Yang2024.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107229776","display_name":"Zhile Yang","orcid":"https://orcid.org/0000-0002-7908-9524"},"institutions":[{"id":"https://openalex.org/I130828816","display_name":"University of Leeds","ror":"https://ror.org/024mrxd33","country_code":"GB","type":"education","lineage":["https://openalex.org/I130828816"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhile Yang","raw_affiliation_strings":["School of Computer Science, University of Leeds, Leeds, U.K"],"raw_orcid":"https://orcid.org/0000-0002-7908-9524","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Leeds, Leeds, U.K","institution_ids":["https://openalex.org/I130828816"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048321911","display_name":"Shangqi Guo","orcid":"https://orcid.org/0000-0003-3181-6881"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shangqi Guo","raw_affiliation_strings":["Department of Precision Instrument, Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-3181-6881","affiliations":[{"raw_affiliation_string":"Department of Precision Instrument, Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084950559","display_name":"Ying Fang","orcid":"https://orcid.org/0000-0002-2490-8502"},"institutions":[{"id":"https://openalex.org/I111753288","display_name":"Fujian Normal University","ror":"https://ror.org/020azk594","country_code":"CN","type":"education","lineage":["https://openalex.org/I111753288"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ying Fang","raw_affiliation_strings":["College of Computer and Cyber Security, Fujian Normal University, Fujian, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer and Cyber Security, Fujian Normal University, Fujian, China","institution_ids":["https://openalex.org/I111753288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048087489","display_name":"Zhaofei Yu","orcid":"https://orcid.org/0000-0002-6913-7553"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaofei Yu","raw_affiliation_strings":["School of Artificial Intelligence, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6913-7553","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Peking University, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5001710920","display_name":"Jian K. Liu","orcid":"https://orcid.org/0000-0002-5391-7213"},"institutions":[{"id":"https://openalex.org/I130828816","display_name":"University of Leeds","ror":"https://ror.org/024mrxd33","country_code":"GB","type":"education","lineage":["https://openalex.org/I130828816"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Jian K. Liu","raw_affiliation_strings":["School of Computer Science, University of Leeds, Leeds, U.K"],"raw_orcid":"https://orcid.org/0000-0002-5391-7213","affiliations":[{"raw_affiliation_string":"School of Computer Science, University of Leeds, Leeds, U.K","institution_ids":["https://openalex.org/I130828816"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8654,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.74291235,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"47","issue":"3","first_page":"1975","last_page":"1990"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11601","display_name":"Neuroscience and Neural Engineering","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/2804","display_name":"Cellular and Molecular Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7159391641616821},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6644003987312317},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6161431074142456},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3740937411785126},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3545679450035095}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7159391641616821},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6644003987312317},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6161431074142456},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3740937411785126},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3545679450035095}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tpami.2024.3511936","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2024.3511936","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:40030447","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/40030447","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null},{"id":"pmh:oai:eprints.whiterose.ac.uk:222690","is_oa":false,"landing_page_url":"https://orcid.org/0000-0002-7908-9524>,","pdf_url":null,"source":{"id":"https://openalex.org/S4306400854","display_name":"White Rose Research Online (University of Leeds, The University of Sheffield, University of York)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2800616092","host_organization_name":"White Rose University Consortium","host_organization_lineage":["https://openalex.org/I2800616092"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:pure.atira.dk:publications/dca78f74-8211-41e3-9c69-55b4372ad98b","is_oa":true,"landing_page_url":"https://research.birmingham.ac.uk/en/publications/dca78f74-8211-41e3-9c69-55b4372ad98b","pdf_url":"https://pure-oai.bham.ac.uk/ws/files/252277110/Yang2024.pdf","source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Yang, Z, Guo, S, Fang, Y, Yu, Z & Liu, J K 2025, 'Spiking Variational Policy Gradient for Brain Inspired Reinforcement Learning', IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 47, no. 3, pp. 1975-1990. https://doi.org/10.1109/TPAMI.2024.3511936","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:publications/dca78f74-8211-41e3-9c69-55b4372ad98b","is_oa":true,"landing_page_url":"https://research.birmingham.ac.uk/en/publications/dca78f74-8211-41e3-9c69-55b4372ad98b","pdf_url":"https://pure-oai.bham.ac.uk/ws/files/252277110/Yang2024.pdf","source":{"id":"https://openalex.org/S4306402634","display_name":"University of Birmingham Research Portal (University of Birmingham)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79619799","host_organization_name":"University of Birmingham","host_organization_lineage":["https://openalex.org/I79619799"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Yang, Z, Guo, S, Fang, Y, Yu, Z & Liu, J K 2025, 'Spiking Variational Policy Gradient for Brain Inspired Reinforcement Learning', IEEE Transactions on Pattern Analysis and Machine Intelligence, vol. 47, no. 3, pp. 1975-1990. https://doi.org/10.1109/TPAMI.2024.3511936","raw_type":"info:eu-repo/semantics/publishedVersion"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10","score":0.41999998688697815}],"awards":[{"id":"https://openalex.org/G3833898155","display_name":null,"funder_award_id":"62088102","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5782943403","display_name":null,"funder_award_id":"20230484362","funder_id":"https://openalex.org/F4320334978","funder_display_name":"Beijing Nova Program"},{"id":"https://openalex.org/G8795465888","display_name":null,"funder_award_id":"62206151","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334978","display_name":"Beijing Nova Program","ror":"https://ror.org/034k14f91"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4405180690.pdf"},"referenced_works_count":80,"referenced_works":["https://openalex.org/W2003400208","https://openalex.org/W2020539709","https://openalex.org/W2038496684","https://openalex.org/W2055523610","https://openalex.org/W2082511574","https://openalex.org/W2126404188","https://openalex.org/W2145339207","https://openalex.org/W2271476098","https://openalex.org/W2292153326","https://openalex.org/W2605102758","https://openalex.org/W2620474507","https://openalex.org/W2736601468","https://openalex.org/W2767899364","https://openalex.org/W2775079417","https://openalex.org/W2794822538","https://openalex.org/W2890344110","https://openalex.org/W2895603185","https://openalex.org/W2898323475","https://openalex.org/W2900163261","https://openalex.org/W2905581998","https://openalex.org/W2913168413","https://openalex.org/W2919115771","https://openalex.org/W2922002199","https://openalex.org/W2946092987","https://openalex.org/W2954718200","https://openalex.org/W2963871073","https://openalex.org/W2963948533","https://openalex.org/W2968865419","https://openalex.org/W2969489794","https://openalex.org/W2978279179","https://openalex.org/W2979879900","https://openalex.org/W2980764939","https://openalex.org/W2987112672","https://openalex.org/W2992534497","https://openalex.org/W3003075284","https://openalex.org/W3029795912","https://openalex.org/W3034201026","https://openalex.org/W3043133474","https://openalex.org/W3044529009","https://openalex.org/W3046208328","https://openalex.org/W3082153440","https://openalex.org/W3091042523","https://openalex.org/W3102087395","https://openalex.org/W3107153805","https://openalex.org/W3110981250","https://openalex.org/W3130821023","https://openalex.org/W3200729277","https://openalex.org/W3201103276","https://openalex.org/W3201870057","https://openalex.org/W3208890675","https://openalex.org/W4214717370","https://openalex.org/W4221145431","https://openalex.org/W4231081240","https://openalex.org/W4282003809","https://openalex.org/W4293052541","https://openalex.org/W4293566037","https://openalex.org/W4294691690","https://openalex.org/W4298857966","https://openalex.org/W4366371396","https://openalex.org/W4387409844","https://openalex.org/W4391791543","https://openalex.org/W6631190155","https://openalex.org/W6637967152","https://openalex.org/W6683006896","https://openalex.org/W6686121877","https://openalex.org/W6741002519","https://openalex.org/W6747106673","https://openalex.org/W6756463683","https://openalex.org/W6759089702","https://openalex.org/W6759312646","https://openalex.org/W6764769363","https://openalex.org/W6766978945","https://openalex.org/W6769808814","https://openalex.org/W6775783182","https://openalex.org/W6780559895","https://openalex.org/W6782642079","https://openalex.org/W6784098896","https://openalex.org/W6784189638","https://openalex.org/W6810161230","https://openalex.org/W6846511701"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W3046775127","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474"],"abstract_inverted_index":{"Recent":[0],"studies":[1],"in":[2],"reinforcement":[3],"learning":[4,11,42,47,75,120],"have":[5],"explored":[6],"brain-inspired":[7],"function":[8,94],"approximators":[9],"and":[10,17,32,44,53,64,98,122,135,161],"algorithms":[12],"to":[13,19,154],"simulate":[14],"brain":[15],"intelligence":[16],"adapt":[18],"neuromorphic":[20],"hardware.":[21],"Among":[22],"these":[23],"approaches,":[24],"reward-modulated":[25],"spike-timing-dependent":[26],"plasticity":[27],"(R-STDP)":[28],"is":[29,88,102],"biologically":[30],"plausible":[31],"energy-efficient,":[33],"but":[34],"suffers":[35],"from":[36,79,90],"a":[37,60,72,105,129],"gap":[38,116],"between":[39,117],"its":[40,51],"local":[41,119],"rules":[43,121],"the":[45,66,80,85,99,109,115,118,123,145],"global":[46,81,110,124],"objectives,":[48],"which":[49],"limits":[50],"performance":[52],"applicability.":[54],"In":[55,126,147],"this":[56],"paper,":[57],"we":[58],"design":[59],"recurrent":[61],"winner-take-all":[62],"network":[63,159],"propose":[65],"spiking":[67],"variational":[68],"policy":[69,82,86,93,100,111],"gradient":[70],"(SVPG),":[71],"new":[73],"R-STDP":[74],"method":[76],"derived":[77,89],"theoretically":[78],"gradient.":[83,112],"Specifically,":[84],"inference":[87],"an":[91],"energy-based":[92],"using":[95],"mean-field":[96],"inference,":[97],"optimization":[101],"based":[103],"on":[104],"last-step":[106],"approximation":[107],"of":[108,157],"These":[113],"fill":[114],"target.":[125],"experiments":[127],"including":[128],"challenging":[130],"ViZDoom":[131],"vision-based":[132],"navigation":[133],"task":[134],"two":[136],"realistic":[137],"robot":[138],"control":[139],"tasks,":[140],"SVPG":[141,149],"successfully":[142],"solves":[143],"all":[144],"tasks.":[146],"addition,":[148],"exhibits":[150],"better":[151],"inherent":[152],"robustness":[153],"various":[155],"kinds":[156],"input,":[158],"parameters,":[160],"environmental":[162],"perturbations":[163],"than":[164],"compared":[165],"methods.":[166]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2025-12-23T23:11:35.936235","created_date":"2025-10-10T00:00:00"}