{"id":"https://openalex.org/W7135075179","doi":"https://doi.org/10.48550/arxiv.2603.10055","title":"Training Language Models via Neural Cellular Automata","display_name":"Training Language Models via Neural Cellular Automata","publication_year":2026,"publication_date":"2026-03-09","ids":{"openalex":"https://openalex.org/W7135075179","doi":"https://doi.org/10.48550/arxiv.2603.10055"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.10055","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10055","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.10055","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061702418","display_name":"Dan P. Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Dan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128853259","display_name":"Seungwook Han","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Han, Seungwook","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101003033","display_name":"Akarsh Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Akarsh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128861492","display_name":"Pulkit Agrawal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agrawal, Pulkit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.48159998655319214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.48159998655319214,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.060600001364946365,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.03909999877214432,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.641700029373169},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.6414999961853027},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.5616999864578247},{"id":"https://openalex.org/keywords/cellular-automaton","display_name":"Cellular automaton","score":0.5449000000953674},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5184000134468079},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.44620001316070557},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.42160001397132874},{"id":"https://openalex.org/keywords/automaton","display_name":"Automaton","score":0.421099990606308},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.40869998931884766}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8122000098228455},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.641700029373169},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.6414999961853027},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.5616999864578247},{"id":"https://openalex.org/C35527583","wikidata":"https://www.wikidata.org/wiki/Q189156","display_name":"Cellular automaton","level":2,"score":0.5449000000953674},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5407999753952026},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5184000134468079},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.44620001316070557},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.42160001397132874},{"id":"https://openalex.org/C112505250","wikidata":"https://www.wikidata.org/wiki/Q787116","display_name":"Automaton","level":2,"score":0.421099990606308},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.4099999964237213},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.40869998931884766},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.388700008392334},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.38600000739097595},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.375900000333786},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.35420000553131104},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.34450000524520874},{"id":"https://openalex.org/C94922259","wikidata":"https://www.wikidata.org/wiki/Q33215","display_name":"Constructed language","level":2,"score":0.3278999924659729},{"id":"https://openalex.org/C147168706","wikidata":"https://www.wikidata.org/wiki/Q1457734","display_name":"Recurrent neural network","level":3,"score":0.3206000030040741},{"id":"https://openalex.org/C167822520","wikidata":"https://www.wikidata.org/wiki/Q176452","display_name":"Finite-state machine","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C2993724205","wikidata":"https://www.wikidata.org/wiki/Q315","display_name":"Human language","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.29499998688697815},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.27709999680519104},{"id":"https://openalex.org/C2776175482","wikidata":"https://www.wikidata.org/wiki/Q1195816","display_name":"Transfer (computing)","level":2,"score":0.26969999074935913},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25850000977516174}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.10055","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10055","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.10055","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.10055","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8750911355018616,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Pre-training":[0],"is":[1,10,26,43],"crucial":[2],"for":[3,63],"large":[4],"language":[5,20,45,79,100,123],"models":[6,199],"(LLMs),":[7],"as":[8],"it":[9,28,33],"when":[11],"most":[12,153],"representations":[13],"and":[14,32,75,83,106,140,155,170],"capabilities":[15],"are":[16,151],"acquired.":[17],"However,":[18],"natural":[19,44,78,122],"pre-training":[21],"has":[22],"problems:":[23],"high-quality":[24],"text":[25,172],"finite,":[27],"contains":[29],"human":[30],"biases,":[31],"entangles":[34],"knowledge":[35],"with":[36,127,200],"reasoning.":[37],"This":[38],"raises":[39],"a":[40,194],"fundamental":[41],"question:":[42],"the":[46,152,183],"only":[47,94],"path":[48,195],"to":[49,58,85,104,111,134,186],"intelligence?":[50],"We":[51,89],"propose":[52],"using":[53],"neural":[54],"cellular":[55],"automata":[56],"(NCA)":[57],"generate":[59,86],"synthetic,":[60],"non-linguistic":[61],"data":[62,70],"pre-pre-training":[64,92,117],"LLMs--training":[65],"on":[66,93,118],"synthetic-then-natural":[67],"language.":[68],"NCA":[69,96,158],"exhibits":[71],"rich":[72],"spatiotemporal":[73],"structure":[74],"statistics":[76],"resembling":[77],"while":[80,168],"being":[81],"controllable":[82],"cheap":[84],"at":[87],"scale.":[88],"find":[90,147],"that":[91,148,156],"164M":[95],"tokens":[97,120],"improves":[98],"downstream":[99],"modeling":[101],"by":[102,109,161],"up":[103,110],"6%":[105],"accelerates":[107],"convergence":[108],"1.6x.":[112],"Surprisingly,":[113],"this":[114],"even":[115],"outperforms":[116],"1.6B":[119],"of":[121,182],"from":[124,165],"Common":[125],"Crawl":[126],"more":[128,174,197],"compute.":[129],"These":[130,177],"gains":[131],"also":[132],"transfer":[133],"reasoning":[135],"benchmarks,":[136],"including":[137],"GSM8K,":[138],"HumanEval,":[139],"BigBench-Lite.":[141],"Investigating":[142],"what":[143],"drives":[144],"transfer,":[145],"we":[146],"attention":[149],"layers":[150],"transferable,":[154],"optimal":[157],"complexity":[159],"varies":[160],"domain:":[162],"code":[163],"benefits":[164],"simpler":[166],"dynamics,":[167],"math":[169],"web":[171],"favor":[173],"complex":[175],"ones.":[176],"results":[178],"enable":[179],"systematic":[180],"tuning":[181],"synthetic":[184,202],"distribution":[185],"target":[187],"domains.":[188],"More":[189],"broadly,":[190],"our":[191],"work":[192],"opens":[193],"toward":[196],"efficient":[198],"fully":[201],"pre-training.":[203]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-03-13T00:00:00"}