{"id":"https://openalex.org/W7204604634","doi":"https://doi.org/10.48550/arxiv.2608.26139","title":"Syntax vs. Semantics: How Transformers Learn Deep Dependencies","display_name":"Syntax vs. Semantics: How Transformers Learn Deep Dependencies","publication_year":2026,"publication_date":"2026-06-26","ids":{"openalex":"https://openalex.org/W7204604634","doi":"https://doi.org/10.48550/arxiv.2608.26139"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.26139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26139","pdf_url":"https://arxiv.org/pdf/2608.26139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.26139","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148097293","display_name":"Jiangrui Zhao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Jiangrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101755124","display_name":"Xiaoting Du","orcid":"https://orcid.org/0000-0002-6963-551X","observed_orcids":["https://orcid.org/0000-0002-6963-551X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Xiaoting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8589000105857849,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8589000105857849,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.11219999939203262,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.003800000064074993,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7039999961853027},{"id":"https://openalex.org/keywords/surface-statistics","display_name":"surface statistics","score":0.609000027179718},{"id":"https://openalex.org/keywords/structural-reasoning","display_name":"structural reasoning","score":0.6790000200271606},{"id":"https://openalex.org/keywords/chain-of-thought-prompting","display_name":"chain-of-thought prompting","score":0.8140000104904175},{"id":"https://openalex.org/keywords/contrastive-learning","display_name":"contrastive learning","score":0.7990000247955322},{"id":"https://openalex.org/keywords/variable-binding","display_name":"variable binding","score":0.7749999761581421},{"id":"https://openalex.org/keywords/mechanistic-understanding","display_name":"mechanistic understanding","score":0.3799999952316284}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7538999915122986},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.637499988079071},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5572999715805054},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.4878999888896942},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4832000136375427},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4575999975204468},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4172999858856201},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C12426560","wikidata":"https://www.wikidata.org/wiki/Q189569","display_name":"Basis (linear algebra)","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C90312973","wikidata":"https://www.wikidata.org/wiki/Q7449052","display_name":"Semantic data model","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.25380000472068787},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2535000145435333}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.26139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26139","pdf_url":"https://arxiv.org/pdf/2608.26139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.26139","doi":"https://doi.org/10.48550/arxiv.2608.26139","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.26139","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.26139","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.26139","pdf_url":"https://arxiv.org/pdf/2608.26139","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204604634.pdf","grobid_xml":"https://content.openalex.org/works/W7204604634.grobid-xml"},"referenced_works_count":27,"referenced_works":["https://openalex.org/W2765574772","https://openalex.org/W2939413764","https://openalex.org/W2953337107","https://openalex.org/W2997753998","https://openalex.org/W3093569337","https://openalex.org/W3096565276","https://openalex.org/W3118781290","https://openalex.org/W3154376977","https://openalex.org/W3154449322","https://openalex.org/W3177813494","https://openalex.org/W4226163008","https://openalex.org/W4226434736","https://openalex.org/W4283026156","https://openalex.org/W4297412003","https://openalex.org/W4316135772","https://openalex.org/W4383996906","https://openalex.org/W4385574039","https://openalex.org/W4387687067","https://openalex.org/W4387724046","https://openalex.org/W4400375635","https://openalex.org/W4401660192","https://openalex.org/W4403796339","https://openalex.org/W4406890777","https://openalex.org/W4414385093","https://openalex.org/W4414588852","https://openalex.org/W7116087444","https://openalex.org/W7125979386"],"related_works":["https://openalex.org/W7125979386","https://openalex.org/W7135041004","https://openalex.org/W7219398644","https://openalex.org/W4416020962","https://openalex.org/W7126110420","https://openalex.org/W7134806122","https://openalex.org/W7124930997","https://openalex.org/W4405029746","https://openalex.org/W4403938641","https://openalex.org/W7161132290"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"demonstrate":[3,150],"remarkable":[4],"syntactic":[5],"fluency,":[6],"yet":[7],"the":[8,49,65,89,106,142],"optimization":[9],"dynamics":[10],"governing":[11],"their":[12],"acquisition":[13],"of":[14,67,91],"deep":[15],"semantic":[16,54],"dependencies":[17,55],"remain":[18],"poorly":[19],"understood.":[20],"We":[21,113],"propose":[22,134],"a":[23,32,44,77,85,135],"mechanistic":[24,86],"framework":[25,83],"that":[26,139,151,157,163],"models":[27,125],"this":[28,82,131],"learning":[29,66],"process":[30],"as":[31,76],"competition":[33],"between":[34],"Surface":[35],"Statistics":[36],"and":[37,70],"Deep":[38],"Semantics.":[39],"Our":[40],"theoretical":[41],"analysis":[42],"identifies":[43],"``Gradient":[45],"Starvation\"":[46],"phenomenon":[47],"where":[48],"error":[50],"signals":[51],"for":[52,88],"sparse":[53],"are":[56],"actively":[57],"suppressed":[58],"during":[59],"early":[60],"optimization.":[61],"This":[62],"suppression":[63,107],"impedes":[64],"structural":[68],"reasoning":[69,98],"causes":[71],"its":[72],"emergence":[73],"to":[74,110,123],"manifest":[75],"sudden":[78],"phase":[79],"transition.":[80],"Furthermore,":[81],"offers":[84],"basis":[87],"effectiveness":[90],"Chain-of-Thought":[92],"(CoT)":[93],"strategies.":[94],"By":[95],"externalizing":[96],"intermediate":[97],"steps":[99],"into":[100],"concrete":[101],"tokens,":[102],"CoT":[103],"effectively":[104],"bypasses":[105],"regime":[108],"inherent":[109],"implicit":[111],"reasoning.":[112],"validate":[114],"these":[115],"findings":[116],"across":[117],"scales":[118],"ranging":[119],"from":[120],"toy":[121],"transformers":[122],"production":[124],"(Llama-3.1-8B,":[126],"Qwen2.5-Coder-7B).":[127],"Finally,":[128],"guided":[129],"by":[130],"theory,":[132],"we":[133],"topology-aligned":[136],"contrastive":[137],"objective":[138],"explicitly":[140],"rectifies":[141],"gradient":[143],"geometry.":[144],"Experiments":[145],"on":[146],"variable":[147],"binding":[148],"tasks":[149],"our":[152],"method":[153],"achieves":[154],"an":[155],"improvement":[156],"is":[158],"over":[159],"2x":[160],"larger":[161],"than":[162],"obtained":[164],"via":[165],"standard":[166],"cross-entropy":[167],"fine-tuning.":[168]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-08-29T00:00:00"}