{"id":"https://openalex.org/W7168809821","doi":"https://doi.org/10.48550/arxiv.2607.13491","title":"DeepLoop: Depth Scaling for Looped Transformers","display_name":"DeepLoop: Depth Scaling for Looped Transformers","publication_year":2026,"publication_date":"2026-07-15","ids":{"openalex":"https://openalex.org/W7168809821","doi":"https://doi.org/10.48550/arxiv.2607.13491"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.13491","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.13491","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.13491","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5141016729","display_name":"Shuzhen Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Shuzhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141011537","display_name":"Yifan Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141009986","display_name":"Jiacheng Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Jiacheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5141004135","display_name":"Quanquan Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Quanquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5078914856","display_name":"M Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Mengdi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.22540000081062317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.22540000081062317,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.10610000044107437,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.06319999694824219,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.8148000240325928},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6776999831199646},{"id":"https://openalex.org/keywords/residual","display_name":"Residual","score":0.6769000291824341},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.652899980545044},{"id":"https://openalex.org/keywords/exponent","display_name":"Exponent","score":0.6068999767303467},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.4966999888420105}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.8148000240325928},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6776999831199646},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.6769000291824341},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.652899980545044},{"id":"https://openalex.org/C2780388253","wikidata":"https://www.wikidata.org/wiki/Q5421508","display_name":"Exponent","level":2,"score":0.6068999767303467},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.4966999888420105},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4593000113964081},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4115000069141388},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.3702999949455261},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3686000108718872},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.36570000648498535},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.33629998564720154},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.27889999747276306},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.27309998869895935},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.26829999685287476},{"id":"https://openalex.org/C133770746","wikidata":"https://www.wikidata.org/wiki/Q856535","display_name":"Current transformer","level":4,"score":0.2662000060081482}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.13491","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.13491","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.13491","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.13491","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Looped":[0],"Transformers":[1],"scale":[2],"sequential":[3],"computation":[4],"by":[5,60,81],"applying":[6],"a":[7,45,76,82],"compact":[8],"stack":[9],"of":[10],"physical":[11,117,154],"blocks":[12],"for":[13,133,183],"multiple":[14],"rounds,":[15],"increasing":[16,20],"unrolled":[17,134],"depth":[18,135,167,176],"without":[19],"stored":[21],"parameters.":[22],"This":[23],"reuse":[24],"changes":[25],"the":[26,65,89,97,103,124],"residual-scaling":[27],"problem:":[28],"in":[29,44,64,96],"an":[30],"untied":[31],"Transformer,":[32],"each":[33],"residual":[34,178],"branch":[35],"receives":[36],"and":[37,56,128,131,145,158,162],"applies":[38],"its":[39],"own":[40],"parameter":[41,184],"update,":[42],"whereas":[43],"looped":[46,139],"Transformer":[47],"one":[48],"shared":[49],"update":[50],"aggregates":[51],"gradients":[52],"from":[53,107],"repeated":[54],"visits":[55,63,93],"is":[57,150,156,168],"read":[58],"back":[59],"those":[61],"same":[62],"next":[66],"linearized":[67],"forward":[68],"pass.":[69],"We":[70],"formalize":[71],"this":[72],"tied-depth":[73],"effect":[74],"through":[75],"first-order":[77],"perturbation":[78],"bound":[79,87],"controlled":[80],"visit-alignment":[83],"coefficient":[84],"$\u03ba_R$.":[85],"The":[86,119],"recovers":[88],"DeepNorm":[90,126],"exponent":[91,104],"when":[92,152],"decorrelate,":[94],"but":[95],"conservative":[98],"aligned":[99],"regime":[100],"it":[101],"requires":[102,177],"to":[105,109],"increase":[106],"$1/4$":[108],"$1/2$":[110],"as":[111],"loop":[112],"count":[113],"grows":[114],"at":[115,142],"fixed":[116],"depth.":[118],"resulting":[120],"method,":[121],"\\textbf{DeepLoop},":[122],"keeps":[123],"Post-LN":[125],"architecture":[127],"sets":[129],"$\u03b1=(2N)^{1/2}$":[130],"$\u03b2=(8N)^{-1/2}$":[132],"$N$.":[136],"On":[137],"GPT-style":[138],"language":[140],"models":[141],"GPT-2":[143,146],"small":[144],"medium":[147],"scale,":[148],"DeepLoop":[149],"neutral":[151],"no":[153],"block":[155],"revisited":[157],"improves":[159],"validation":[160],"loss":[161],"downstream":[163],"accuracy":[164],"once":[165],"recurrent":[166,175],"activated.":[169],"These":[170],"results":[171],"show":[172],"that":[173,181],"stable":[174],"scaling":[179],"rules":[180],"account":[182],"visits,":[185],"not":[186],"only":[187],"nominal":[188],"layer":[189],"count.":[190]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-17T00:00:00"}