{"id":"https://openalex.org/W2994716630","doi":null,"title":"Improved memory in recurrent neural networks with sequential non-normal dynamics","display_name":"Improved memory in recurrent neural networks with sequential non-normal dynamics","alternate_titles":[],"publication_year":2020,"publication_date":"2020-04-30","ids":{"openalex":"https://openalex.org/W2994716630","mag":"2994716630"},"language":"en","primary_location":{"id":"mag:2994716630","doi":null,"is_oa":true,"landing_page_url":"https://openreview.net/pdf?id=ryx1wRNFvB","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null},"parent_works":[],"lineage":["https://openalex.org/W2994716630"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://openreview.net/pdf?id=ryx1wRNFvB","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031940826","display_name":"Emin Orhan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Emin Orhan","raw_affiliation_strings":["New York University, New York, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University, New York, United States","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057055829","display_name":"Xaq Pitkow","orcid":"https://orcid.org/0000-0001-6376-329X","observed_orcids":["https://orcid.org/0000-0001-6376-329X"]},"institutions":[{"id":"https://openalex.org/I181547552","display_name":"Baylor College of Medicine","ror":"https://ror.org/02pttbw34","country_code":"US","type":"education","lineage":["https://openalex.org/I181547552","https://openalex.org/I2801539370"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Xaq Pitkow","raw_affiliation_strings":["Baylor College of Medicine, Houston, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Baylor College of Medicine, Houston, United States","institution_ids":["https://openalex.org/I181547552"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3989,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.60684891,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.961899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.961899995803833,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.007899999618530273,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.0017999999690800905,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.9929999709129333},{"id":"https://openalex.org/keywords/vanishing-and-exploding-gradients","display_name":"vanishing and exploding gradients","score":0.7670000195503235},{"id":"https://openalex.org/keywords/signal-to-noise-ratio-maximization","display_name":"signal-to-noise ratio maximization","score":0.8199999928474426},{"id":"https://openalex.org/keywords/orthogonal-initialization","display_name":"orthogonal initialization","score":0.5979999899864197},{"id":"https://openalex.org/keywords/sequential-dynamics","display_name":"sequential dynamics","score":0.48899999260902405},{"id":"https://openalex.org/keywords/nonlinearity","display_name":"nonlinearity","score":0.45399999618530273}],"concepts":[{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.8197773098945618},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7136440277099609},{"id":"https://openalex.org/C47559304","wikidata":"https://www.wikidata.org/wiki/Q1702189","display_name":"Orthogonalization","level":2,"score":0.6467012166976929},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6151176691055298},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5001280307769775},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.42981451749801636},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.4271070957183838},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38001787662506104},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.36716943979263306},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.337568998336792},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.32550400495529175},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2994716630","doi":null,"is_oa":true,"landing_page_url":"https://openreview.net/pdf?id=ryx1wRNFvB","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null}],"best_oa_location":{"id":"mag:2994716630","doi":null,"is_oa":true,"landing_page_url":"https://openreview.net/pdf?id=ryx1wRNFvB","pdf_url":null,"source":{"id":"https://openalex.org/S4306419637","display_name":"International Conference on Learning Representations","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"International Conference on Learning Representations","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W194249466","https://openalex.org/W1800356822","https://openalex.org/W2046305855","https://openalex.org/W2064675550","https://openalex.org/W2095705004","https://openalex.org/W2105398397","https://openalex.org/W2107878631","https://openalex.org/W2157331557","https://openalex.org/W2175402905","https://openalex.org/W2176412452","https://openalex.org/W2194775991","https://openalex.org/W2295845209","https://openalex.org/W2547718140","https://openalex.org/W2743945814","https://openalex.org/W2792376130","https://openalex.org/W2792764867","https://openalex.org/W2808197691","https://openalex.org/W2947963487","https://openalex.org/W2963241221","https://openalex.org/W2963570896","https://openalex.org/W2963755273"],"related_works":["https://openalex.org/W2947963487","https://openalex.org/W2562512778","https://openalex.org/W2997081737","https://openalex.org/W2752658941","https://openalex.org/W2099000454","https://openalex.org/W2243290847","https://openalex.org/W2943592883","https://openalex.org/W2557389081","https://openalex.org/W3192210946","https://openalex.org/W2737836580"],"abstract_inverted_index":{"Training":[0],"recurrent":[1,49,162,198,240],"neural":[2],"networks":[3,163,175],"(RNNs)":[4],"is":[5,66,142],"a":[6,16,104,196,217],"hard":[7],"problem":[8,17,41],"due":[9],"to":[10,45,120,133],"degeneracies":[11],"in":[12,124,158,201,216,234],"the":[13,21,48,56,73,85,93,101,125,135,144,149,159,166,188],"optimization":[14],"landscape,":[15],"also":[18,223],"known":[19],"as":[20],"vanishing/exploding":[22,86],"gradients":[23,87],"problem.":[24,88],"Short":[25],"of":[26,47,72,96,103,127,148,190,220],"designing":[27],"new":[28],"RNN":[29],"architectures,":[30],"various":[31],"methods":[32,65],"that":[33,67,113,157,164,209],"have":[34],"been":[35],"proposed":[36],"for":[37,226],"dealing":[38],"with":[39,84,195,238],"this":[40,90,180,184],"usually":[42],"boil":[43],"down":[44],"orthogonalization":[46],"dynamics,":[50],"either":[51],"at":[52],"initialization":[53],"or":[54],"during":[55],"entire":[57],"training":[58],"period.":[59],"The":[60],"basic":[61],"motivation":[62],"behind":[63],"these":[64],"orthogonal":[68,106,174,214,239],"transformations":[69,107,115],"are":[70,176],"isometries":[71],"Euclidean":[74],"space,":[75],"hence":[76],"they":[77],"preserve":[78,110],"(Euclidean)":[79],"norms":[80],"and":[81,98,173,229],"effectively":[82],"deal":[83],"However,":[89],"idea":[91],"ignores":[92],"crucial":[94],"effects":[95],"non-linearity":[97],"noise.":[99],"In":[100],"presence":[102,126],"non-linearity,":[105],"no":[108],"longer":[109],"norms,":[111],"suggesting":[112],"alternative":[114],"might":[116],"be":[117,134],"better":[118],"suited":[119],"non-linear":[121],"networks.":[122],"Moreover,":[123],"noise,":[128],"norm":[129],"preservation":[130],"itself":[131],"ceases":[132],"ideal":[136],"objective.":[137],"A":[138],"more":[139],"sensible":[140],"objective":[141],"maximizing":[143],"signal-to-noise":[145],"ratio":[146],"(SNR)":[147],"propagated":[150],"signal":[151],"instead.":[152],"Previous":[153],"work":[154],"has":[155],"shown":[156],"linear":[160],"case,":[161],"maximize":[165],"SNR":[167],"display":[168],"strongly":[169],"non-normal,":[170],"sequential":[171,202],"dynamics":[172],"highly":[177],"suboptimal":[178],"by":[179,183],"measure.":[181],"Motivated":[182],"finding,":[185],"we":[186],"investigate":[187],"potential":[189],"non-normal":[191,197,210],"RNNs,":[192],"i.e.":[193],"RNNs":[194,211,236],"connectivity":[199,241],"matrix,":[200],"processing":[203],"tasks.":[204],"Our":[205],"experimental":[206],"results":[207],"show":[208],"outperform":[212],"their":[213],"counterparts":[215],"diverse":[218],"range":[219],"benchmarks.":[221],"We":[222],"find":[224],"evidence":[225],"increased":[227],"non-normality":[228],"hidden":[230],"chain-like":[231],"feedforward":[232],"structures":[233],"trained":[235],"initialized":[237],"matrices.":[242]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W2994716630","counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":2}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2019-12-26T00:00:00"}
