{"id":"https://openalex.org/W7147076727","doi":"https://doi.org/10.5281/zenodo.19358184","title":"Ep. 182: Beyond the Transformer: The New AI Architecture Wars","display_name":"Ep. 182: Beyond the Transformer: The New AI Architecture Wars","publication_year":2026,"publication_date":"2026-01-06","ids":{"openalex":"https://openalex.org/W7147076727","doi":"https://doi.org/10.5281/zenodo.19358184"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.19358184","doi":"https://doi.org/10.5281/zenodo.19358184","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.19358184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"volume":null,"type":"other","is_traditional":false,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.19358184","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5128325680","display_name":"Daniel Rosehill","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rosehill, Daniel","raw_affiliation_strings":["My Weird Prompts"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"My Weird Prompts","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128791034","display_name":"Gemini 3.1 (Flash)","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210090411","display_name":"Google DeepMind (United Kingdom)","ror":"https://ror.org/00971b260","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210090411","https://openalex.org/I4210128969"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Gemini 3.1 (Flash)","raw_affiliation_strings":["Google DeepMind"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind","institution_ids":["https://openalex.org/I4210090411"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5128778162","display_name":"Chatterbox TTS","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chatterbox TTS","raw_affiliation_strings":["Resemble AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Resemble AI","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8852999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8852999806404114,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.07890000194311142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.6959999799728394},{"id":"https://openalex.org/keywords/self-attention","display_name":"self-attention","score":0.6959999799728394},{"id":"https://openalex.org/keywords/state-space-models","display_name":"state-space model","score":0.7889999747276306},{"id":"https://openalex.org/keywords/mamba","display_name":"Mamba","score":0.8230000138282776},{"id":"https://openalex.org/keywords/rwkv","display_name":"RWKV","score":0.8980000019073486},{"id":"https://openalex.org/keywords/hybrid-model","display_name":"hybrid model","score":0.4519999921321869},{"id":"https://openalex.org/keywords/long-context-modeling","display_name":"long-context modeling","score":0.4189999997615814},{"id":"https://openalex.org/keywords/linear-scaling","display_name":"linear scaling","score":0.6470000147819519},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7279999852180481},{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.7649999856948853},{"id":"https://openalex.org/keywords/key-value-cache","display_name":"key-value cache","score":0.609000027179718},{"id":"https://openalex.org/keywords/memory-efficiency","display_name":"memory efficiency","score":0.48899999260902405},{"id":"https://openalex.org/keywords/language-modeling","display_name":"language modeling","score":0.35199999809265137}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5619000196456909},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5422000288963318},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5390999913215637},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.44670000672340393},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.40290001034736633},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.39800000190734863},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.36239999532699585},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3564000129699707},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2944999933242798},{"id":"https://openalex.org/C174998907","wikidata":"https://www.wikidata.org/wiki/Q357662","display_name":"Work in process","level":2,"score":0.27880001068115234},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.26930001378059387},{"id":"https://openalex.org/C171686336","wikidata":"https://www.wikidata.org/wiki/Q3532085","display_name":"Topic model","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.25609999895095825},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.25450000166893005}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.19358184","doi":"https://doi.org/10.5281/zenodo.19358184","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.19358184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},{"id":"doi:10.5281/zenodo.19358185","doi":"https://doi.org/10.5281/zenodo.19358185","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.19358185","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"}],"best_oa_location":{"id":"doi:10.5281/zenodo.19358184","doi":"https://doi.org/10.5281/zenodo.19358184","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.19358184","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Audiovisual"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7147339591","https://openalex.org/W7147464633","https://openalex.org/W7147328418","https://openalex.org/W7196929164","https://openalex.org/W4401271669","https://openalex.org/W6891955290","https://openalex.org/W7222754568","https://openalex.org/W7201874312","https://openalex.org/W7077933215","https://openalex.org/W7125218161"],"abstract_inverted_index":{"Episode":[0],"summary:":[1],"For":[2,99],"years,":[3],"the":[4,8,31,55,61,76,82,103,111,123,130,133,145,149,164,169,174,177,211,216,240,251,271,277,280,302,322,333,354,377,422,427,434,474,514,518,555,560,563,617,620,630,635,663,671,674,739,742,748,758,779,787],"transformer":[5,134,342,589,743],"has":[6,356,776],"been":[7],"undisputed":[9],"king":[10],"of":[11,34,58,63,80,113,126,132,213,265,270,282,306,362,376,408,458,500,533,551,565,604,619,634,639,673,694,738,760],"AI,":[12],"but":[13,343,703,712],"its":[14,21,107,503],"\"quadratic":[15,146],"bottleneck\"":[16,147],"is":[17,86,136,171,204,279,365,480,506,562,785],"starting":[18],"to":[19,66,138,154,182,192,247,255,299,331,335,399,418,426,476,541,580,606,666,724,763],"show":[20],"age.":[22],"In":[23,75,509],"this":[24,726],"episode,":[25],"Herman":[26,118,162,324,568,669,751],"and":[27,40,46,119,148,219,259,318,410,441,483,502,524,569,584,684,701,752,771,778],"Corn":[28,120,570,753],"dive":[29],"into":[30],"2026":[32],"landscape":[33,79],"alternative":[35],"architectures":[36,152,764],"like":[37,312,340,345,489,652,717],"Mamba,":[38],"RWKV,":[39],"x-LSTM":[41,319],"that":[42,207,389,463,537,578,733,757,765],"promise":[43],"linear":[44,507,511],"scaling":[45],"infinite":[47],"context.":[48],"Discover":[49],"how":[50,92],"hybrid":[51,566,642],"models":[52,65,95,181,246,311,599,643],"are":[53,97,644,722],"combining":[54],"reasoning":[56,610,628],"power":[57,305],"attention":[59,178,583,605],"with":[60,160,395,591],"efficiency":[62],"state-space":[64,585],"redefine":[67],"what's":[68],"possible":[69],"in":[70,91,122,189,338,403,677,782,791],"language":[71,94],"modeling.":[72],"Show":[73],"Notes":[74],"rapidly":[77],"evolving":[78],"2026,":[81,640],"artificial":[83],"intelligence":[84],"community":[85],"witnessing":[87],"a":[88,101,190,199,214,225,341,357,385,446,453,490,495,510,543,601,667,688,729,745],"fundamental":[89],"shift":[90],"large":[93],"(LLMs)":[96],"built.":[98],"nearly":[100],"decade,":[102],"\"Transformer\"":[104],"architecture\u2014defined":[105],"by":[106,166,185,276,438,727],"self-attention":[108],"mechanism\u2014was":[109],"considered":[110],"pinnacle":[112],"machine":[114,792],"learning.":[115],"However,":[116,309],"as":[117,233,384],"discuss":[121,571],"latest":[124],"episode":[125,561,664],"*My":[127],"Weird":[128],"Prompts*,":[129],"era":[131],"monoculture":[135,775],"coming":[137],"an":[139,346,573],"end.":[140],"The":[141,158,263,406,414,497,549,659,774],"discussion":[142,165,749],"centers":[143],"on":[144,301,535,649],"innovative":[150],"new":[151,310],"designed":[153],"shatter":[155],"it.":[156],"###":[157,262,405,546,658],"Problem":[159],"Attention":[161],"opens":[163],"explaining":[167],"why":[168],"industry":[170],"looking":[172],"beyond":[173],"transformer.":[175,545],"While":[176,741],"mechanism":[179],"allows":[180,473,527,625],"understand":[183],"context":[184,529],"comparing":[186],"every":[187,193],"word":[188],"sequence":[191],"other":[194],"word,":[195],"it":[196,364,521,755],"comes":[197],"at":[198,707],"steep":[200],"price.":[201],"This":[202,223,350,472,526,623],"process":[203],"\"quadratic,\"":[205],"meaning":[206],"if":[208],"you":[209],"double":[210],"length":[212],"text,":[215,252],"computational":[217],"work":[218],"memory":[220,359,632],"required":[221,243],"quadruple.":[222],"creates":[224],"massive":[226,237,631],"barrier":[227],"for":[228,244,528,616,626],"processing":[229,304,366,451],"long-form":[230],"content,":[231],"such":[232],"entire":[234],"books":[235],"or":[236,369],"codebases.":[238],"Furthermore,":[239],"\"key-value":[241],"cache\"":[242],"these":[245,641,695],"function":[248],"grows":[249],"alongside":[250],"eventually":[253],"leading":[254],"astronomical":[256],"energy":[257],"costs":[258],"hardware":[260,536],"limitations.":[261],"Renaissance":[264],"Recurrent":[266,283],"Neural":[267,284],"Networks":[268,285],"One":[269],"most":[272,423,556],"surprising":[273],"developments":[274],"discussed":[275],"brothers":[278],"comeback":[281],"(RNNs).":[286],"Historically,":[287,687],"RNNs":[288,709],"were":[289],"sidelined":[290],"because":[291],"they":[292],"processed":[293],"information":[294,466,479],"sequentially,":[295],"making":[296,520],"them":[297],"impossible":[298],"train":[300],"parallel":[303,339],"modern":[307,396],"GPUs.":[308],"RWKV":[313],"(Receptance":[314],"Weighted":[315],"Key":[316],"Value)":[317],"have":[320],"changed":[321],"game.":[323],"highlights":[325],"RWKV-7,":[326],"which":[327,478,484,654],"uses":[328],"clever":[329],"mathematics":[330],"allow":[332],"model":[334,355,475,689],"be":[336,393,486],"trained":[337],"act":[344],"RNN":[347],"during":[348,494],"use.":[349],"\"shapeshifting\"":[351],"ability":[352],"means":[353],"constant":[358],"footprint,":[360],"regardless":[361],"whether":[363],"ten":[367,370],"words":[368],"thousand.":[371],"Similarly,":[372],"x-LTSM\u2014a":[373],"modernized":[374],"version":[375],"classic":[378],"Long":[379],"Short-Term":[380],"Memory":[381],"network\u2014has":[382],"emerged":[383],"\"Pareto-dominant\"":[386],"force,":[387],"proving":[388],"old":[390],"concepts":[391],"can":[392,485,766],"revitalized":[394],"\"warp":[397],"drives\"":[398],"outperform":[400],"traditional":[401],"transformers":[402],"efficiency.":[404],"Rise":[407],"Mamba":[409,435,444,468,501,592,614],"State":[411,429],"Space":[412,430],"Models":[413,431],"conversation":[415],"then":[416],"turns":[417],"what":[419],"many":[420],"consider":[421],"significant":[424],"challenger":[425],"transformer:":[428],"(SSMs),":[432],"specifically":[433],"architecture.":[436],"Developed":[437],"Albert":[439],"Gu":[440],"Tri":[442],"Dao,":[443],"represents":[445],"departure":[447],"from":[448,559,575],"discrete":[449],"text":[450],"toward":[452],"more":[454],"continuous,":[455],"fluid":[456],"stream":[457],"information.":[459],"Unlike":[460],"earlier":[461],"SSMs":[462],"treated":[464],"all":[465,735],"equally,":[467],"introduced":[469],"\"selective\"":[470],"mechanisms.":[471],"decide":[477],"worth":[481],"remembering":[482],"discarded,":[487],"much":[488],"human":[491],"taking":[492],"notes":[493],"lecture.":[496],"primary":[498],"advantage":[499],"successor,":[504],"Mamba-2,":[505],"scaling.":[508],"system,":[512],"doubling":[513],"input":[515],"only":[516,691],"doubles":[517],"work,":[519],"incredibly":[522],"fast":[523],"efficient.":[525],"windows":[530],"spanning":[531],"millions":[532],"tokens":[534],"would":[538],"typically":[539],"struggle":[540],"run":[542,772],"standard":[544],"Hybrid":[547],"Vigor:":[548],"Best":[550],"Both":[552],"Worlds":[553],"Perhaps":[554],"practical":[557],"insight":[558],"rise":[564],"architectures.":[567],"Jamba,":[572],"architecture":[574,784],"AI21":[576],"Labs":[577],"refuses":[579],"choose":[581],"between":[582,750],"models.":[586],"By":[587],"interleaving":[588],"layers":[590,615],"layers,":[593],"Jamba":[594],"achieves":[595],"\"hybrid":[596],"vigor.\"":[597],"These":[598],"use":[600],"small":[602],"amount":[603],"handle":[607],"complex,":[608],"non-linear":[609],"tasks":[611],"while":[612],"utilizing":[613],"bulk":[618],"heavy":[621],"lifting.":[622],"approach":[624],"high-quality":[627],"without":[629],"overhead":[633],"KV":[636],"cache.":[637],"As":[638,662],"enabling":[645],"sophisticated":[646],"\"reasoning\"":[647],"capabilities":[648],"consumer":[650],"devices":[651],"smartphones,":[653],"was":[655],"previously":[656],"unthinkable.":[657],"Impossible":[660],"Trinity":[661],"draws":[665],"close,":[668],"introduces":[670],"concept":[672],"\"Impossible":[675],"Trinity\"":[676],"LLM":[678],"design:":[679],"training":[680],"parallelization,":[681],"low-latency":[682,705,711],"inference,":[683],"high":[685],"performance.":[686],"could":[690],"achieve":[692],"two":[693],"three":[696,736],"goals.":[697],"Transformers":[698],"offered":[699,710],"parallelization":[700],"performance":[702],"lacked":[704,713],"inference":[706],"scale.":[708],"parallelization.":[714],"New":[715],"contenders":[716],"Microsoft's":[718],"RetNet":[719],"(Retentive":[720],"Networks)":[721],"attempting":[723],"solve":[725],"providing":[728],"unified":[730],"theoretical":[731],"framework":[732],"hits":[734],"points":[737],"triangle.":[740],"remains":[744],"powerful":[746],"tool,":[747],"makes":[754],"clear":[756],"future":[759],"AI":[761,783],"belongs":[762],"scale":[767],"linearly,":[768],"think":[769],"deeply,":[770],"efficiently.":[773],"broken,":[777],"resulting":[780],"diversity":[781],"driving":[786],"next":[788],"great":[789],"leap":[790],"intelligence.":[793],"Listen":[794],"online:":[795],"https://myweirdprompts.com/episode/ai-architectures-beyond-transformers":[796]},"counts_by_year":[],"updated_date":"2026-10-07T09:51:55.459714","created_date":"2026-04-02T00:00:00"}