{"id":"https://openalex.org/W4294691690","doi":"https://doi.org/10.1109/tcyb.2022.3198259","title":"Human-Level Control Through Directly Trained Deep Spiking <i>Q</i>-Networks","display_name":"Human-Level Control Through Directly Trained Deep Spiking <i>Q</i>-Networks","publication_year":2022,"publication_date":"2022-09-05","ids":{"openalex":"https://openalex.org/W4294691690","doi":"https://doi.org/10.1109/tcyb.2022.3198259","pmid":"https://pubmed.ncbi.nlm.nih.gov/36063509"},"language":"en","primary_location":{"id":"doi:10.1109/tcyb.2022.3198259","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2022.3198259","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5003393482","display_name":"Guisong Liu","orcid":"https://orcid.org/0000-0003-2360-0466"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]},{"id":"https://openalex.org/I204831749","display_name":"Southwestern University of Finance and Economics","ror":"https://ror.org/04ewct822","country_code":"CN","type":"education","lineage":["https://openalex.org/I204831749"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guisong Liu","raw_affiliation_strings":["School of Computing and Artificial Intelligence, Southwestern University of Finance and Economics, Chengdu, China","School of Computer Science, Zhongshan Institute, University of Electronic Science and Technology of China, Zhongshan, China"],"raw_orcid":"https://orcid.org/0000-0003-2360-0466","affiliations":[{"raw_affiliation_string":"School of Computing and Artificial Intelligence, Southwestern University of Finance and Economics, Chengdu, China","institution_ids":["https://openalex.org/I204831749"]},{"raw_affiliation_string":"School of Computer Science, Zhongshan Institute, University of Electronic Science and Technology of China, Zhongshan, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060734662","display_name":"Wenjie Deng","orcid":"https://orcid.org/0000-0002-5530-5413"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Deng","raw_affiliation_strings":["School of Computer Science and Engineering, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-5530-5413","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029314634","display_name":"Xiurui Xie","orcid":"https://orcid.org/0000-0002-3720-4379"},"institutions":[{"id":"https://openalex.org/I150229711","display_name":"University of Electronic Science and Technology of China","ror":"https://ror.org/04qr3zq92","country_code":"CN","type":"education","lineage":["https://openalex.org/I150229711"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiurui Xie","raw_affiliation_strings":["School of Computer Science and Engineering, University of Electronic Science and Technology of China, Chengdu, China"],"raw_orcid":"https://orcid.org/0000-0002-3720-4379","affiliations":[{"raw_affiliation_string":"School of Computer Science and Engineering, University of Electronic Science and Technology of China, Chengdu, China","institution_ids":["https://openalex.org/I150229711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069393874","display_name":"Li Huang","orcid":"https://orcid.org/0009-0002-1093-8302"},"institutions":[{"id":"https://openalex.org/I204831749","display_name":"Southwestern University of Finance and Economics","ror":"https://ror.org/04ewct822","country_code":"CN","type":"education","lineage":["https://openalex.org/I204831749"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li Huang","raw_affiliation_strings":["School of Computing and Artificial Intelligence, Southwestern University of Finance and Economics, Chengdu, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Artificial Intelligence, Southwestern University of Finance and Economics, Chengdu, China","institution_ids":["https://openalex.org/I204831749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057282687","display_name":"Huajin Tang","orcid":"https://orcid.org/0000-0002-9857-1543"},"institutions":[{"id":"https://openalex.org/I168879160","display_name":"Zhejiang University of Science and Technology","ror":"https://ror.org/05mx0wr29","country_code":"CN","type":"education","lineage":["https://openalex.org/I168879160"]},{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huajin Tang","raw_affiliation_strings":["College of Computer Science and Technology, Zhejiang University, Hangzhou, China","Institute of Artificial Intelligence, Zhejiang Laboratory, Hangzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-9857-1543","affiliations":[{"raw_affiliation_string":"College of Computer Science and Technology, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I168879160"]},{"raw_affiliation_string":"Institute of Artificial Intelligence, Zhejiang Laboratory, Hangzhou, China","institution_ids":["https://openalex.org/I4210123185"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.5339,"has_fulltext":false,"cited_by_count":53,"citation_normalized_percentile":{"value":0.93592625,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"53","issue":"11","first_page":"7187","last_page":"7198"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/spiking-neural-network","display_name":"Spiking neural network","score":0.8991174697875977},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8023198843002319},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7813560366630554},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6939681768417358},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6588221192359924},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.5797793865203857},{"id":"https://openalex.org/keywords/neuromorphic-engineering","display_name":"Neuromorphic engineering","score":0.5591414570808411},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5526865720748901},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.49121397733688354},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.4169304370880127},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3662649393081665}],"concepts":[{"id":"https://openalex.org/C11731999","wikidata":"https://www.wikidata.org/wiki/Q9067355","display_name":"Spiking neural network","level":3,"score":0.8991174697875977},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8023198843002319},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7813560366630554},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6939681768417358},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6588221192359924},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.5797793865203857},{"id":"https://openalex.org/C151927369","wikidata":"https://www.wikidata.org/wiki/Q1981312","display_name":"Neuromorphic engineering","level":3,"score":0.5591414570808411},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5526865720748901},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.49121397733688354},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.4169304370880127},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3662649393081665},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tcyb.2022.3198259","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tcyb.2022.3198259","pdf_url":null,"source":{"id":"https://openalex.org/S4210191041","display_name":"IEEE Transactions on Cybernetics","issn_l":"2168-2267","issn":["2168-2267","2168-2275"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Cybernetics","raw_type":"journal-article"},{"id":"pmid:36063509","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/36063509","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on cybernetics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Affordable and clean energy","score":0.8999999761581421,"id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G1563777489","display_name":null,"funder_award_id":"2021A1515011866","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"}],"funders":[{"id":"https://openalex.org/F4320321921","display_name":"Natural Science Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W1645800954","https://openalex.org/W1963689209","https://openalex.org/W2138913040","https://openalex.org/W2145339207","https://openalex.org/W2159951683","https://openalex.org/W2257979135","https://openalex.org/W2316725540","https://openalex.org/W2519051215","https://openalex.org/W2588977645","https://openalex.org/W2761873684","https://openalex.org/W2767899364","https://openalex.org/W2767993539","https://openalex.org/W2775079417","https://openalex.org/W2783525259","https://openalex.org/W2786930404","https://openalex.org/W2892077605","https://openalex.org/W2897070044","https://openalex.org/W2898323475","https://openalex.org/W2902907165","https://openalex.org/W2964338223","https://openalex.org/W2969489794","https://openalex.org/W2974328520","https://openalex.org/W2980764939","https://openalex.org/W2982160266","https://openalex.org/W2982316857","https://openalex.org/W2984844508","https://openalex.org/W2989431475","https://openalex.org/W2998119008","https://openalex.org/W3005486893","https://openalex.org/W3010162417","https://openalex.org/W3010748916","https://openalex.org/W3022627268","https://openalex.org/W3082274886","https://openalex.org/W3093146707","https://openalex.org/W3102750118","https://openalex.org/W3176730514","https://openalex.org/W3183892796","https://openalex.org/W3186962187","https://openalex.org/W3189897383","https://openalex.org/W3196571547","https://openalex.org/W3200729277","https://openalex.org/W3208922547","https://openalex.org/W4298857966","https://openalex.org/W6637967152","https://openalex.org/W6780181723","https://openalex.org/W6780628335","https://openalex.org/W6784189638","https://openalex.org/W6798240405"],"related_works":["https://openalex.org/W3089892344","https://openalex.org/W4313442939","https://openalex.org/W4386227293","https://openalex.org/W4372267706","https://openalex.org/W4288055417","https://openalex.org/W4287758233","https://openalex.org/W2960220682","https://openalex.org/W4205804651","https://openalex.org/W3136467750","https://openalex.org/W2885510266"],"abstract_inverted_index":{"As":[0],"the":[1,29,44,48,52,80,99,104,128,136,151,159,171],"third-generation":[2],"neural":[3,6],"networks,":[4],"spiking":[5,23,53,63,95],"networks":[7],"(SNNs)":[8],"have":[9,115],"great":[10],"potential":[11],"on":[12,34,79,118,166],"neuromorphic":[13],"hardware":[14],"because":[15],"of":[16,51,106,138,143,153],"their":[17],"high":[18],"energy":[19,148],"efficiency.":[20,149],"However,":[21],"deep":[22,62,86],"reinforcement":[24,30],"learning":[25,31,96],"(DSRL),":[26],"that":[27],"is,":[28],"(RL)":[32],"based":[33,78],"SNNs,":[35],"is":[36,158],"still":[37],"in":[38,67,110,141],"its":[39],"preliminary":[40],"stage":[41],"due":[42],"to":[43,123,162],"binary":[45],"output":[46],"and":[47,85,147],"nondifferentiable":[49],"property":[50],"function.":[54],"To":[55,150],"address":[56],"these":[57],"issues,":[58],"we":[59,71,91],"propose":[60,72],"a":[61,73,93],"Q":[64,87],"-network":[65,88],"(DSQN)":[66],"this":[68],"article.":[69],"Specifically,":[70],"directly":[74,172],"trained":[75,173],"DSRL":[76],"architecture":[77],"leaky":[81],"integrate-and-fire":[82],"(LIF)":[83],"neurons":[84,109],"(DQN).":[89],"Then,":[90],"adapt":[92],"direct":[94],"algorithm":[97],"for":[98],"DSQN.":[100],"We":[101],"further":[102],"demonstrate":[103,135],"advantages":[105],"using":[107],"LIF":[108],"DSQN":[111],"theoretically.":[112],"Comprehensive":[113],"experiments":[114],"been":[116],"conducted":[117],"17":[119],"top-performing":[120],"Atari":[121,168],"games":[122,169],"compare":[124],"our":[125,139,154,156],"method":[126,140],"with":[127,170],"state-of-the-art":[129,164],"conversion":[130],"method.":[131],"The":[132],"experimental":[133],"results":[134],"superiority":[137],"terms":[142],"performance,":[144],"stability,":[145],"generalization":[146],"best":[152],"knowledge,":[155],"work":[157],"first":[160],"one":[161],"achieve":[163],"performance":[165],"multiple":[167],"SNN.":[174]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":16},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2022-09-06T00:00:00"}