{"id":"https://openalex.org/W7208765302","doi":"https://doi.org/10.48550/arxiv.2609.02986","title":"Modern Transformers Are Implicit Hybrids: From Functional Differentiation to Principled Hybrid Architecture Design","display_name":"Modern Transformers Are Implicit Hybrids: From Functional Differentiation to Principled Hybrid Architecture Design","publication_year":2026,"publication_date":"2026-09-02","ids":{"openalex":"https://openalex.org/W7208765302","doi":"https://doi.org/10.48550/arxiv.2609.02986"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.02986","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02986","pdf_url":"https://arxiv.org/pdf/2609.02986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.02986","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5150032004","display_name":"Runlin Shi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shi, Runlin","raw_affiliation_strings":["Institute of Automation , Chinese Academy of Sciences , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation , Chinese Academy of Sciences , Beijing , China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081601428","display_name":"Bojian Yin","orcid":"https://orcid.org/0000-0002-5074-4337","observed_orcids":["https://orcid.org/0000-0002-5074-4337"]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yin, Bojian","raw_affiliation_strings":["Institute of Automation , Chinese Academy of Sciences , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation , Chinese Academy of Sciences , Beijing , China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5150027443","display_name":"Guoqi Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Guoqi","raw_affiliation_strings":["Institute of Automation , Chinese Academy of Sciences , Beijing , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Automation , Chinese Academy of Sciences , Beijing , China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9351000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.029500000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.009700000286102295,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/full-attention","display_name":"full attention","score":0.8769999742507935},{"id":"https://openalex.org/keywords/linear-attention","display_name":"linear attention","score":0.996999979019165},{"id":"https://openalex.org/keywords/rotary-positional-encoding","display_name":"rotary positional encoding","score":0.7310000061988831},{"id":"https://openalex.org/keywords/long-context-language-modeling","display_name":"long-context language modeling","score":0.46299999952316284},{"id":"https://openalex.org/keywords/commonsense-reasoning","display_name":"commonsense reasoning","score":0.6460000276565552}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7098000049591064},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6514000296592712},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5860999822616577},{"id":"https://openalex.org/C2780719617","wikidata":"https://www.wikidata.org/wiki/Q1030752","display_name":"Salient","level":2,"score":0.5022000074386597},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.49000000953674316},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.41839998960494995},{"id":"https://openalex.org/C162269090","wikidata":"https://www.wikidata.org/wiki/Q1156047","display_name":"Rope","level":2,"score":0.36390000581741333},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32760000228881836},{"id":"https://openalex.org/C132459708","wikidata":"https://www.wikidata.org/wiki/Q744069","display_name":"Extrapolation","level":2,"score":0.30630001425743103},{"id":"https://openalex.org/C98025372","wikidata":"https://www.wikidata.org/wiki/Q477538","display_name":"Systems architecture","level":3,"score":0.29260000586509705},{"id":"https://openalex.org/C62235348","wikidata":"https://www.wikidata.org/wiki/Q3264234","display_name":"Functional requirement","level":2,"score":0.2903999984264374},{"id":"https://openalex.org/C20854674","wikidata":"https://www.wikidata.org/wiki/Q4386060","display_name":"Cognitive architecture","level":3,"score":0.2816999852657318},{"id":"https://openalex.org/C169900460","wikidata":"https://www.wikidata.org/wiki/Q2200417","display_name":"Cognition","level":2,"score":0.2689000070095062},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C50897621","wikidata":"https://www.wikidata.org/wiki/Q2665508","display_name":"Hybrid system","level":2,"score":0.25189998745918274},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.2517000138759613},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.25049999356269836}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.02986","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02986","pdf_url":"https://arxiv.org/pdf/2609.02986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.02986","doi":"https://doi.org/10.48550/arxiv.2609.02986","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.02986","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.02986","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.02986","pdf_url":"https://arxiv.org/pdf/2609.02986","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7208765302.pdf","grobid_xml":"https://content.openalex.org/works/W7208765302.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7165398008","https://openalex.org/W7166873788","https://openalex.org/W4415317893","https://openalex.org/W7168102439","https://openalex.org/W7156343422","https://openalex.org/W4417261040","https://openalex.org/W7126262583","https://openalex.org/W7214941653","https://openalex.org/W4416102280","https://openalex.org/W4416617107"],"abstract_inverted_index":{"Hybrid":[0,156],"architectures":[1],"combining":[2],"Full":[3],"Attention":[4,8],"(FA)":[5],"and":[6,59,73,77,85,123,139,165,182,188,197,206],"Linear":[7],"(LA)":[9],"are":[10],"increasingly":[11],"prominent,":[12],"yet":[13],"their":[14],"allocation":[15],"remains":[16],"heuristic.":[17],"We":[18,151],"seek":[19],"an":[20,172],"evidence-grounded":[21],"basis":[22],"in":[23,154],"head-level":[24],"functional":[25],"organization":[26],"learned":[27],"by":[28,89],"RoPE-based":[29],"Transformers.":[30],"Behavioral":[31],"probes":[32],"do":[33],"not":[34],"yield":[35],"a":[36,55,80,90,116,198,215],"complete":[37,81],"taxonomy,":[38],"so":[39],"we":[40,105],"propose":[41],"two":[42,125],"intervention":[43],"metrics:":[44],"RoPE":[45,60],"Frequency":[46],"Importance":[47],"Score":[48],"(RFIS),":[49],"measuring":[50],"how":[51],"each":[52],"frequency":[53],"affects":[54],"head's":[56],"attention":[57],"distribution,":[58],"Positional":[61,110],"Dependence":[62],"(RPD),":[63],"isolating":[64],"dependence":[65],"on":[66],"rotary":[67],"positional":[68,86,103,127,169],"modulation.":[69],"On":[70],"Qwen3-series":[71],"models":[72],"Llama3.1,":[74],"RFIS":[75],"suggests":[76,115],"RPD":[78],"verifies":[79],"taxonomy":[82],"of":[83,119],"retrieval":[84,164,187],"heads":[87],"separated":[88],"salient":[91],"mid-low-frequency":[92],"band.":[93],"Controlled":[94],"Transformers":[95],"show":[96],"that":[97],"this":[98],"boundary":[99],"follows":[100],"the":[101,108],"training-length":[102],"scale;":[104],"term":[106],"it":[107],"Global":[109],"Band":[111],"(GPBand).":[112],"The":[113],"analysis":[114],"potential":[117],"cause":[118],"zero-shot":[120,191],"length-extrapolation":[121],"failure":[122],"yields":[124],"principles:":[126],"modeling":[128,181],"should":[129,142],"operate":[130],"only":[131],"locally,":[132],"with":[133,148],"global":[134,163],"access":[135],"through":[136],"position-independent":[137],"retrieval;":[138],"both":[140,204],"functions":[141],"be":[143],"assigned":[144],"at":[145],"head":[146],"granularity":[147],"layer-specific":[149],"allocation.":[150],"instantiate":[152],"them":[153],"Head-wise":[155],"Architecture":[157],"(HwH),":[158],"using":[159],"NoPE":[160],"FA":[161],"for":[162,167],"LA":[166],"local":[168],"modeling.":[170],"With":[171],"FA-to-LA":[173],"ratio":[174],"below":[175],"1:3,":[176],"HwH":[177],"retains":[178],"strong":[179],"language":[180],"commonsense":[183],"reasoning":[184],"while":[185],"improving":[186],"substantially":[189],"strengthening":[190],"long-context":[192],"extrapolation":[193],"over":[194],"Transformer,":[195],"LA,":[196],"layer-wise":[199],"hybrid":[200,211],"baseline.":[201],"Ablations":[202],"validate":[203],"principles":[205],"component":[207],"roles,":[208],"highlighting":[209],"principled":[210],"architecture":[212],"design":[213],"as":[214],"promising":[216],"route":[217],"toward":[218],"future":[219],"foundation":[220],"models.":[221]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-09-05T00:00:00"}