{"id":"https://openalex.org/W7168261255","doi":"https://doi.org/10.48550/arxiv.2607.11875","title":"Invariant Learning Dynamics of Transformers in Inductive Reasoning Tasks","display_name":"Invariant Learning Dynamics of Transformers in Inductive Reasoning Tasks","publication_year":2026,"publication_date":"2026-07-13","ids":{"openalex":"https://openalex.org/W7168261255","doi":"https://doi.org/10.48550/arxiv.2607.11875"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.11875","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11875","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.11875","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5140695648","display_name":"Tiberiu Musat","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Musat, Tiberiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140625101","display_name":"Tiago Pimentel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pimentel, Tiago","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140583020","display_name":"Nicholas Zucchet","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zucchet, Nicolas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140746466","display_name":"Thomas Hofmann","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hofmann, Thomas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.21490000188350677,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.21490000188350677,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10656","display_name":"Child and Animal Learning Development","score":0.11060000211000443,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08739999681711197,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6790000200271606},{"id":"https://openalex.org/keywords/inductive-bias","display_name":"Inductive bias","score":0.6172999739646912},{"id":"https://openalex.org/keywords/inductive-reasoning","display_name":"Inductive reasoning","score":0.5598999857902527},{"id":"https://openalex.org/keywords/invariant","display_name":"Invariant (physics)","score":0.4884999990463257},{"id":"https://openalex.org/keywords/inductive-logic-programming","display_name":"Inductive logic programming","score":0.38420000672340393},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"Multi-task learning","score":0.3531000018119812}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6790000200271606},{"id":"https://openalex.org/C197352929","wikidata":"https://www.wikidata.org/wiki/Q1074074","display_name":"Inductive bias","level":4,"score":0.6172999739646912},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5871000289916992},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5835999846458435},{"id":"https://openalex.org/C21563000","wikidata":"https://www.wikidata.org/wiki/Q484511","display_name":"Inductive reasoning","level":2,"score":0.5598999857902527},{"id":"https://openalex.org/C190470478","wikidata":"https://www.wikidata.org/wiki/Q2370229","display_name":"Invariant (physics)","level":2,"score":0.4884999990463257},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38940000534057617},{"id":"https://openalex.org/C2779382394","wikidata":"https://www.wikidata.org/wiki/Q1464197","display_name":"Inductive logic programming","level":2,"score":0.38420000672340393},{"id":"https://openalex.org/C28006648","wikidata":"https://www.wikidata.org/wiki/Q6934509","display_name":"Multi-task learning","level":3,"score":0.3531000018119812},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.28839999437332153},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C97364631","wikidata":"https://www.wikidata.org/wiki/Q484284","display_name":"Deductive reasoning","level":2,"score":0.2669000029563904},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.26350000500679016},{"id":"https://openalex.org/C79379906","wikidata":"https://www.wikidata.org/wiki/Q3174497","display_name":"Dynamical systems theory","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2581000030040741}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.11875","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11875","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.11875","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.11875","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7255652546882629}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0],"present":[1],"a":[2,35,73,88,166,172,175],"theoretical":[3,100],"framework":[4],"to":[5,30,72,150],"explain":[6],"the":[7,48,63,82,115,128,140,145],"emergence":[8],"of":[9,38,66,90,96,178],"inductive":[10,39],"reasoning":[11],"abilities":[12],"in":[13,47,158],"Transformer":[14,21],"language":[15],"models.":[16,160],"While":[17],"previous":[18],"works":[19],"on":[20],"learning":[22,83],"dynamics":[23,65,84],"have":[24,155],"so":[25],"far":[26],"been":[27,156],"mostly":[28],"tied":[29],"specific":[31],"tasks,":[32],"we":[33,59,109,122,137,170],"study":[34,123],"generalized":[36],"class":[37],"tasks":[40,45],"that":[41,62,139],"unifies":[42],"several":[43],"synthetic":[44],"known":[46],"literature,":[49],"including":[50],"in-context":[51,118],"n-grams":[52],"and":[53,101,119,136],"multi-hop":[54],"reasoning.":[55],"In":[56],"this":[57,80,107],"class,":[58],"theoretically":[60],"prove":[61],"training":[64],"attention":[67],"models":[68],"can":[69,147],"be":[70,148],"confined":[71],"highly":[74],"interpretable,":[75],"low-dimensional":[76,167],"invariant":[77],"manifold.":[78],"On":[79],"manifold,":[81],"are":[85,134],"captured":[86],"by":[87],"handful":[89],"interpretable":[91],"coordinates":[92],"rather":[93],"than":[94],"millions":[95],"parameters,":[97],"making":[98],"both":[99],"empirical":[102],"analysis":[103],"more":[104],"tractable.":[105],"Using":[106],"framework,":[108],"characterize":[110],"how":[111,124,179],"data":[112],"statistics":[113],"govern":[114],"competition":[116],"between":[117],"in-weights":[120],"learning,":[121],"random":[125],"initializations":[126],"determine":[127],"`winning'":[129],"circuit":[130,163],"when":[131],"multiple":[132],"solutions":[133],"possible,":[135],"demonstrate":[138],"coordinate":[141],"frame":[142],"associated":[143],"with":[144],"manifold":[146],"used":[149],"automatically":[151],"detect":[152],"which":[153],"circuits":[154],"learned":[157],"trained":[159],"By":[161],"casting":[162],"formation":[164],"as":[165],"dynamical":[168],"phenomenon,":[169],"take":[171],"step":[173],"toward":[174],"predictive":[176],"theory":[177],"Transformers":[180],"learn.":[181]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-15T00:00:00"}