{"id":"https://openalex.org/W7137125774","doi":"https://doi.org/10.48550/arxiv.2603.12541","title":"As Language Models Scale, Low-order Linear Depth Dynamics Emerge","display_name":"As Language Models Scale, Low-order Linear Depth Dynamics Emerge","publication_year":2026,"publication_date":"2026-03-13","ids":{"openalex":"https://openalex.org/W7137125774","doi":"https://doi.org/10.48550/arxiv.2603.12541"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2603.12541","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.12541","pdf_url":"https://arxiv.org/pdf/2603.12541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2603.12541","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5080503570","display_name":"Buddhika Nettasinghe","orcid":"https://orcid.org/0000-0002-6070-892X","observed_orcids":["https://orcid.org/0000-0002-6070-892X"]},"institutions":[{"id":"https://openalex.org/I126307644","display_name":"University of Iowa","ror":"https://ror.org/036jqmy94","country_code":"US","type":"education","lineage":["https://openalex.org/I126307644"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nettasinghe, Buddhika","raw_affiliation_strings":["University of Iowa , Iowa City , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Iowa , Iowa City , USA","institution_ids":["https://openalex.org/I126307644"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012419698","display_name":"Geethu Joseph","orcid":"https://orcid.org/0000-0002-5289-5403","observed_orcids":["https://orcid.org/0000-0002-5289-5403"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Joseph, Geethu","raw_affiliation_strings":["Delft University of Technology , Delft , Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft University of Technology , Delft , Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.5917999744415283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.5917999744415283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2728999853134155,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.059700001031160355,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9279999732971191},{"id":"https://openalex.org/keywords/model-scaling","display_name":"model scaling","score":0.7260000109672546},{"id":"https://openalex.org/keywords/gpt-2","display_name":"GPT-2","score":0.8320000171661377},{"id":"https://openalex.org/keywords/additive-disturbances","display_name":"additive disturbances","score":0.597000002861023},{"id":"https://openalex.org/keywords/toxicity","display_name":"toxicity","score":0.3050000071525574},{"id":"https://openalex.org/keywords/irony","display_name":"irony","score":0.7710000276565552},{"id":"https://openalex.org/keywords/hate-speech","display_name":"hate speech","score":0.9610000252723694},{"id":"https://openalex.org/keywords/sentiment-analysis","display_name":"sentiment analysis","score":0.8700000047683716}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6001999974250793},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.5856000185012817},{"id":"https://openalex.org/C158622935","wikidata":"https://www.wikidata.org/wiki/Q660848","display_name":"Nonlinear system","level":2,"score":0.5702999830245972},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.5444999933242798},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5432000160217285},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.5228000283241272},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.5145999789237976},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.4684999883174896},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41280001401901245},{"id":"https://openalex.org/C170122806","wikidata":"https://www.wikidata.org/wiki/Q1914828","display_name":"Linear scale","level":2,"score":0.39640000462532043},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36320000886917114},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C41587187","wikidata":"https://www.wikidata.org/wiki/Q1501882","display_name":"Generalized linear model","level":2,"score":0.3580999970436096},{"id":"https://openalex.org/C6802819","wikidata":"https://www.wikidata.org/wiki/Q1072174","display_name":"Linear system","level":2,"score":0.33390000462532043},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3303999900817871},{"id":"https://openalex.org/C70519679","wikidata":"https://www.wikidata.org/wiki/Q6666755","display_name":"Log-linear model","level":3,"score":0.29910001158714294},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29589998722076416},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.2939999997615814},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.2858000099658966},{"id":"https://openalex.org/C77405623","wikidata":"https://www.wikidata.org/wiki/Q598451","display_name":"System dynamics","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C3019152653","wikidata":"https://www.wikidata.org/wiki/Q186290","display_name":"Linear relationship","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.25099998712539673}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2603.12541","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.12541","pdf_url":"https://arxiv.org/pdf/2603.12541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2603.12541","doi":"https://doi.org/10.48550/arxiv.2603.12541","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.12541","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2603.12541","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2603.12541","pdf_url":"https://arxiv.org/pdf/2603.12541","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/16","score":0.7229360938072205,"display_name":"Peace, justice, and strong institutions"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7137125774.pdf","grobid_xml":"https://content.openalex.org/works/W7137125774.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1,118],"models":[2,119],"are":[3],"often":[4],"viewed":[5],"as":[6,12,117],"high-dimensional":[7],"nonlinear":[8],"systems":[9],"and":[10,36,134],"treated":[11],"black":[13],"boxes.":[14],"Here,":[15],"we":[16],"show":[17],"that":[18,98,116],"transformer":[19],"depth":[20,123],"dynamics":[21,124],"admit":[22],"accurate":[23],"low-order":[24,121],"linear":[25,40,74,91,122],"surrogates":[26],"within":[27,126],"context.":[28],"Across":[29],"tasks":[30],"including":[31],"toxicity,":[32],"irony,":[33],"hate":[34],"speech":[35],"sentiment,":[37],"a":[38,67,72,129],"32-dimensional":[39],"surrogate":[41,92],"reproduces":[42],"the":[43,54,78,87,109],"layerwise":[44],"sensitivity":[45],"profile":[46],"of":[47],"GPT-2-large":[48],"with":[49,77,83],"near-perfect":[50],"agreement,":[51],"capturing":[52],"how":[53],"final":[55],"output":[56],"shifts":[57],"under":[58],"additive":[59],"injections":[60],"at":[61],"each":[62],"layer.":[63],"We":[64],"then":[65],"uncover":[66],"surprising":[68],"scaling":[69],"principle:":[70],"for":[71,132],"fixed-order":[73],"surrogate,":[75],"agreement":[76],"full":[79,110],"model":[80,84],"improves":[81],"monotonically":[82],"size":[85],"across":[86],"GPT-2":[88],"family.":[89],"This":[90],"also":[93],"enables":[94],"principled":[95],"multi-layer":[96],"interventions":[97],"require":[99],"less":[100],"energy":[101],"than":[102],"standard":[103],"heuristic":[104],"schedules":[105],"when":[106],"applied":[107],"to":[108],"model.":[111],"Together,":[112],"our":[113],"results":[114],"reveal":[115],"scale,":[120],"emerge":[125],"contexts,":[127],"offering":[128],"systems-theoretic":[130],"foundation":[131],"analyzing":[133],"controlling":[135],"them.":[136]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2026-03-17T00:00:00"}