{"id":"https://openalex.org/W4417136618","doi":"https://doi.org/10.48550/arxiv.2506.07919","title":"Uncovering the Computational Roles of Nonlinearity in Sequence Modeling Using Almost-Linear RNNs","display_name":"Uncovering the Computational Roles of Nonlinearity in Sequence Modeling Using Almost-Linear RNNs","publication_year":2025,"publication_date":"2025-06-09","ids":{"openalex":"https://openalex.org/W4417136618","doi":"https://doi.org/10.48550/arxiv.2506.07919"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2506.07919","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.07919","pdf_url":"https://arxiv.org/pdf/2506.07919","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2506.07919","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5013541854","display_name":"Manuel Brenner","orcid":"https://orcid.org/0009-0000-6672-4042"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brenner, Manuel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5012894482","display_name":"Georgia Koppe","orcid":"https://orcid.org/0000-0003-2941-9238"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Koppe, Georgia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.18129999935626984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.18129999935626984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.16459999978542328,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.06449999660253525,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.7876999974250793},{"id":"https://openalex.org/keywords/nonlinear-system","display_name":"Nonlinear system","score":0.7649999856948853},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.5612999796867371},{"id":"https://openalex.org/keywords/computational-model","display_name":"Computational model","score":0.46639999747276306},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.4652000069618225},{"id":"https://openalex.org/keywords/recurrent-neural-network","display_name":"Recurrent neural network","score":0.4530999958515167},{"id":"https://openalex.org/keywords/linear-model","display_name":"Linear model","score":0.4325000047683716}],"concepts":[{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.7876999974250793},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.7649999856948853},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.628000020980835},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.5612999796867371},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.47369998693466187},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.46639999747276306},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.4652000069618225},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.4530999958515167},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.4325000047683716},{"id":"https://openalex.org/C22157029","wikidata":"https://www.wikidata.org/wiki/Q17080460","display_name":"Nonlinear system identification","level":4,"score":0.3993000090122223},{"id":"https://openalex.org/C17095337","wikidata":"https://www.wikidata.org/wiki/Q2375229","display_name":"Piecewise linear function","level":2,"score":0.3885999917984009},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3677000105381012},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.34279999136924744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33059999346733093},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.3262999951839447},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3046000003814697},{"id":"https://openalex.org/C6802819","wikidata":"https://www.wikidata.org/wiki/Q1072174","display_name":"Linear system","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C160824197","wikidata":"https://www.wikidata.org/wiki/Q2071054","display_name":"Linear approximation","level":3,"score":0.26249998807907104}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2506.07919","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.07919","pdf_url":"https://arxiv.org/pdf/2506.07919","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:pure.mpg.de:item_3689241","is_oa":true,"landing_page_url":"http://hdl.handle.net/21.11116/0000-0012-73BC-7","pdf_url":null,"source":{"id":"https://openalex.org/S4306400654","display_name":"MPG.PuRe (Max Planck Society)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I149899117","host_organization_name":"Max Planck Society","host_organization_lineage":["https://openalex.org/I149899117"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Open Review","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2506.07919","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2506.07919","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2506.07919","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2506.07919","pdf_url":"https://arxiv.org/pdf/2506.07919","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Sequence":[0],"modeling":[1,118],"tasks":[2,194],"across":[3,109],"domains":[4],"such":[5,141],"as":[6,142,184],"natural":[7],"language":[8],"processing,":[9],"time":[10],"series":[11],"forecasting,":[12],"and":[13,70,93,112,124,146,166,176,234],"control":[14],"require":[15,195],"learning":[16],"complex":[17],"input-output":[18],"mappings.":[19],"Nonlinear":[20],"recurrence":[21,87],"is":[22,46,67,221],"theoretically":[23],"required":[24],"for":[25,217],"universal":[26],"approximation":[27],"of":[28,43,59,138,227],"sequence-to-sequence":[29],"functions,":[30],"yet":[31],"linear":[32,99,134,156,199],"recurrent":[33,62,228],"models":[34,203],"often":[35,204],"prove":[36],"surprisingly":[37],"effective.":[38],"This":[39],"raises":[40],"the":[41,56,107,131,225],"question":[42],"when":[44,65,193],"nonlinearity":[45,60,88,161,182,220],"truly":[47],"required.":[48],"We":[49,75,105,128],"present":[50],"a":[51,120,125,185,214],"framework":[52,108],"to":[53,89],"systematically":[54],"dissect":[55],"functional":[57],"role":[58],"in":[61,173,189],"networks,":[63],"identifying":[64,218],"it":[66,73],"computationally":[68],"necessary":[69],"what":[71],"mechanisms":[72,103],"enables.":[74],"address":[76],"this":[77],"using":[78],"Almost":[79],"Linear":[80],"Recurrent":[81],"Neural":[82],"Networks":[83],"(AL-RNNs),":[84],"which":[85],"allow":[86],"be":[90],"gradually":[91],"attenuated":[92],"decompose":[94],"network":[95],"dynamics":[96],"into":[97],"analyzable":[98],"regimes,":[100,200],"making":[101],"computational":[102,139,178],"explicit.":[104],"illustrate":[106],"diverse":[110],"synthetic":[111],"real-world":[113],"tasks,":[114,159],"including":[115],"classic":[116],"sequence":[117],"benchmarks,":[119],"neuroscientific":[121],"stimulus-selection":[122],"task,":[123],"multi-task":[126,174],"suite.":[127],"demonstrate":[129],"how":[130],"AL-RNN's":[132],"piecewise":[133],"structure":[135],"enables":[136],"identification":[137],"primitives":[140],"gating,":[143],"rule-based":[144],"integration,":[145],"memory-dependent":[147],"transients,":[148],"revealing":[149],"that":[150,230],"these":[151],"operations":[152],"emerge":[153],"within":[154],"predominantly":[155],"backbones.":[157],"Across":[158],"sparse":[160,181],"improves":[162],"interpretability":[163],"by":[164],"reducing":[165],"localizing":[167],"nonlinear":[168,202,209],"computations,":[169],"promotes":[170],"shared":[171],"representations":[172],"settings,":[175],"reduces":[177],"cost.":[179],"Moreover,":[180],"acts":[183],"useful":[186],"inductive":[187],"bias:":[188],"low-data":[190],"regimes":[191],"or":[192,206],"discrete":[196],"switching":[197],"between":[198],"sparsely":[201],"match":[205],"exceed":[207],"fully":[208],"architectures.":[210],"Our":[211],"findings":[212],"provide":[213],"principled":[215],"approach":[216],"where":[219],"functionally":[222],"necessary,":[223],"guiding":[224],"design":[226],"architectures":[229],"balance":[231],"performance,":[232],"efficiency,":[233],"interpretability.":[235]},"counts_by_year":[],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-10T00:00:00"}