{"id":"https://openalex.org/W7225252587","doi":"https://doi.org/10.48550/arxiv.2611.03998","title":"Dory1.0-4B: A Looped Normalized-GPT","display_name":"Dory1.0-4B: A Looped Normalized-GPT","publication_year":2026,"publication_date":"2026-11-02","ids":{"openalex":"https://openalex.org/W7225252587","doi":"https://doi.org/10.48550/arxiv.2611.03998"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2611.03998","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03998","pdf_url":"https://arxiv.org/pdf/2611.03998","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2611.03998","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012385682","display_name":"Maxim Fishman","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fishman, Maxim","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160696344","display_name":"Simeng Sun","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Simeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102839771","display_name":"Shantanu Acharya","orcid":"https://orcid.org/0000-0002-9652-2991","observed_orcids":["https://orcid.org/0000-0002-9652-2991"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Acharya, Shantanu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074277807","display_name":"Faisal Ladhak","orcid":"https://orcid.org/0009-0009-5730-3866","observed_orcids":["https://orcid.org/0009-0009-5730-3866"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ladhak, Faisal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017964819","display_name":"Brian Chmiel","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chmiel, Brian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5158525926","display_name":"Santiago Akle Serano","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Serano, Santiago Akle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091675091","display_name":"Ron Banner","orcid":"https://orcid.org/0000-0002-7333-7830","observed_orcids":["https://orcid.org/0000-0002-7333-7830"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Banner, Ron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032957280","display_name":"Boris Ginsburg","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ginsburg, Boris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5069691915","display_name":"Krishna C. Puvvada","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Puvvada, Krishna C.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8817999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8817999958992004,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09650000184774399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.004100000020116568,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sliding-window-attention","display_name":"sliding window attention","score":0.8560000061988831},{"id":"https://openalex.org/keywords/token-efficiency","display_name":"token efficiency","score":0.39399999380111694},{"id":"https://openalex.org/keywords/kv-cache-memory-reduction","display_name":"KV cache memory reduction","score":0.7409999966621399},{"id":"https://openalex.org/keywords/long-context-generation","display_name":"long-context generation","score":0.4560000002384186},{"id":"https://openalex.org/keywords/reinforcement-learning-with-verifiable-rewards","display_name":"reinforcement learning with verifiable rewards","score":0.7940000295639038},{"id":"https://openalex.org/keywords/model-quantization","display_name":"model quantization","score":0.5019999742507935}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6269999742507935},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.46939998865127563},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.4471000134944916},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4300000071525574},{"id":"https://openalex.org/C2777210771","wikidata":"https://www.wikidata.org/wiki/Q4927124","display_name":"Block (permutation group theory)","level":2,"score":0.42820000648498535},{"id":"https://openalex.org/C184670325","wikidata":"https://www.wikidata.org/wiki/Q512604","display_name":"Loop (graph theory)","level":2,"score":0.38679999113082886},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.3815999925136566},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3801000118255615},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.36090001463890076},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35580000281333923},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.3547999858856201},{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.3522999882698059},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.3522000014781952},{"id":"https://openalex.org/C186886427","wikidata":"https://www.wikidata.org/wiki/Q5441213","display_name":"Feedback loop","level":2,"score":0.34119999408721924},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.33059999346733093},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.2996000051498413},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.29809999465942383},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2831000089645386},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.2759999930858612},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.2750999927520752},{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.2727999985218048},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2624000012874603},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.25060001015663147}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2611.03998","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03998","pdf_url":"https://arxiv.org/pdf/2611.03998","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2611.03998","doi":"https://doi.org/10.48550/arxiv.2611.03998","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2611.03998","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2611.03998","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2611.03998","pdf_url":"https://arxiv.org/pdf/2611.03998","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7225252587.pdf","grobid_xml":"https://content.openalex.org/works/W7225252587.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W2606964149","https://openalex.org/W2866343820","https://openalex.org/W2950700477","https://openalex.org/W3083410900","https://openalex.org/W3134642945","https://openalex.org/W3195140297","https://openalex.org/W4379089709","https://openalex.org/W4387561453","https://openalex.org/W4391631327","https://openalex.org/W4405714299","https://openalex.org/W4406840771","https://openalex.org/W4407310640","https://openalex.org/W4414850566","https://openalex.org/W4415796274","https://openalex.org/W4415798388","https://openalex.org/W4416361213","https://openalex.org/W4417136556","https://openalex.org/W4417156119","https://openalex.org/W7102671386","https://openalex.org/W7125193990","https://openalex.org/W7125508931","https://openalex.org/W7133198585","https://openalex.org/W7164944187","https://openalex.org/W7168280501","https://openalex.org/W7221554986"],"related_works":["https://openalex.org/W7219155330","https://openalex.org/W7221554986","https://openalex.org/W7208734415","https://openalex.org/W7224027830","https://openalex.org/W7213266125","https://openalex.org/W7220926987","https://openalex.org/W7221941876","https://openalex.org/W7225105315","https://openalex.org/W7223896114","https://openalex.org/W7220986529"],"abstract_inverted_index":{"We":[0,61,228],"introduce":[1],"Dory1.0-4B,":[2],"a":[3,12,24,40,56,105,214],"4-billion-parameter":[4],"looped":[5],"(latent-recurrent)":[6],"language":[7],"model":[8,92,216],"with":[9,29,100,104,122,165,174],"elastic":[10],"depth:":[11],"single":[13],"checkpoint":[14],"serves":[15],"different":[16],"inference":[17],"compute":[18],"budgets.":[19],"Dory1.0-4B":[20,63,137,159,196],"is":[21,181],"built":[22],"on":[23,64,126,151,162,188],"normalized":[25],"Transformer":[26],"(nGPT)":[27],"backbone":[28],"interleaved":[30],"sliding-window":[31],"and":[32,44,77,97,129,147,199,234],"full-attention":[33],"layers,":[34],"organized":[35],"into":[36],"an":[37,45],"input":[38],"block,":[39,43],"shared":[41],"recurrent":[42,51],"output":[46],"block.":[47],"At":[48],"inference,":[49],"the":[50,82,91,117,123,178,203,225,230],"block":[52],"can":[53],"be":[54],"applied":[55],"variable":[57],"number":[58],"of":[59,213,224],"times.":[60],"pre-trained":[62],"6":[65],"trillion":[66,71],"tokens.":[67,87],"The":[68],"first":[69],"4.5":[70],"tokens":[72,172],"were":[73],"trained":[74],"in":[75,183],"NVFP4,":[76,233],"additional":[78],"pre-training":[79],"then":[80],"extended":[81],"context":[83],"length":[84],"to":[85,113,211],"256K":[86],"Next,":[88],"we":[89],"post-trained":[90],"using":[93],"Supervised":[94],"Fine-Tuning":[95],"(SFT)":[96],"Reinforcement":[98],"Learning":[99],"Verifiable":[101],"Rewards":[102],"(RLVR),":[103],"difficulty-aware":[106],"loop":[107,118,219],"curriculum":[108],"that":[109,212],"routes":[110],"harder":[111],"prompts":[112],"deeper":[114],"loops.":[115],"Increasing":[116],"count":[119,220],"improves":[120],"accuracy,":[121],"largest":[124],"gains":[125],"coding,":[127],"agentic,":[128],"long-context":[130],"benchmarks.":[131,155],"With":[132],"two":[133],"or":[134],"more":[135,160],"loops,":[136,167],"outperforms":[138],"larger":[139],"models":[140],"such":[141],"as":[142,177],"Qwen3.5-9B,":[143],"Gemma":[144,148],"4":[145,149],"12B,":[146],"26B-A4B":[150],"average":[152],"across":[153],"12":[154],"Looping":[156],"also":[157],"makes":[158],"token-efficient":[161],"agentic":[163],"benchmarks:":[164],"four":[166],"it":[168],"generates":[169],"1.3-2.5x":[170],"fewer":[171],"than":[173,187],"one":[175],"loop,":[176,205],"extra":[179],"computation":[180],"spent":[182],"latent":[184],"space":[185],"rather":[186],"longer":[189],"outputs.":[190],"To":[191],"reduce":[192],"memory":[193],"during":[194],"looping,":[195],"caches":[197],"keys":[198],"values":[200],"only":[201],"from":[202],"last":[204],"keeping":[206],"its":[207],"KV-cache":[208],"footprint":[209],"equal":[210],"non-looped":[215],"at":[217],"any":[218],"while":[221],"retaining":[222],"most":[223],"accuracy":[226],"gains.":[227],"open-source":[229],"BF16,":[231],"MXFP8,":[232],"GGUF":[235],"quantized":[236],"checkpoints.":[237]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-11-05T00:00:00"}