{"id":"https://openalex.org/W4383473945","doi":"https://doi.org/10.48550/arxiv.2307.02486","title":"LongNet: Scaling Transformers to 1,000,000,000 Tokens","display_name":"LongNet: Scaling Transformers to 1,000,000,000 Tokens","publication_year":2023,"publication_date":"2023-07-05","ids":{"openalex":"https://openalex.org/W4383473945","doi":"https://doi.org/10.48550/arxiv.2307.02486"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2307.02486","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02486","pdf_url":"https://arxiv.org/pdf/2307.02486","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.02486","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018372874","display_name":"Jiayu Ding","orcid":"https://orcid.org/0000-0002-6562-2754","observed_orcids":["https://orcid.org/0000-0002-6562-2754"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Ding, Jiayu","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113130010","display_name":"Shuming Ma","orcid":"https://orcid.org/0000-0003-1091-1206","observed_orcids":["https://orcid.org/0000-0003-1091-1206"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Ma, Shuming","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5132824881","display_name":"Li Dong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Dong, Li","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100343569","display_name":"Xingxing Zhang","orcid":"https://orcid.org/0000-0003-4012-3796","observed_orcids":["https://orcid.org/0000-0003-4012-3796","https://orcid.org/0009-0007-5028-9585"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zhang, Xingxing","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061624006","display_name":"Shaohan Huang","orcid":"https://orcid.org/0000-0003-4324-6337","observed_orcids":["https://orcid.org/0000-0003-4324-6337"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Huang, Shaohan","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100370086","display_name":"Wenhui Wang","orcid":"https://orcid.org/0009-0009-5840-022X","observed_orcids":["https://orcid.org/0009-0009-5840-022X","https://orcid.org/0000-0003-3258-1885"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Wang, Wenhui","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zheng, Nanning","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN","US"],"is_corresponding":false,"raw_author_name":"Zheng, Nanning","raw_affiliation_strings":["Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research \u2663 Xi'an Jiaotong University https ://aka. / GeneralAI","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I87445476"]}]},{"author_position":"last","author":{"id":null,"display_name":"Wei, Furu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Furu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":251,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9150000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9150000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05889999866485596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.004699999932199717,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/long-sequence-modeling","display_name":"long sequence modeling","score":0.7459999918937683},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.593999981880188},{"id":"https://openalex.org/keywords/linear-computational-complexity","display_name":"linear computational complexity","score":0.7360000014305115},{"id":"https://openalex.org/keywords/distributed-training","display_name":"distributed training","score":0.7940000295639038},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7110000252723694},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.30399999022483826}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7349466681480408},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6760512590408325},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5668022036552429},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.5502955913543701},{"id":"https://openalex.org/C39927690","wikidata":"https://www.wikidata.org/wiki/Q11197","display_name":"Logarithm","level":2,"score":0.5435587763786316},{"id":"https://openalex.org/C35639132","wikidata":"https://www.wikidata.org/wiki/Q7452468","display_name":"Sequence labeling","level":3,"score":0.5426973700523376},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.531410276889801},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4923020601272583},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.45286980271339417},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.415332555770874},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3883898854255676},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.31000977754592896},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1493062674999237},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2307.02486","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02486","pdf_url":"https://arxiv.org/pdf/2307.02486","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2307.02486","doi":"https://doi.org/10.48550/arxiv.2307.02486","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2307.02486","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.02486","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.02486","pdf_url":"https://arxiv.org/pdf/2307.02486","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4383473945.pdf","grobid_xml":"https://content.openalex.org/works/W4383473945.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7123909967","https://openalex.org/W3093260394","https://openalex.org/W7214393466","https://openalex.org/W4399452094","https://openalex.org/W4414829925","https://openalex.org/W7165515573","https://openalex.org/W3181262653","https://openalex.org/W7133231433","https://openalex.org/W4416355669","https://openalex.org/W7108662860"],"abstract_inverted_index":{"Scaling":[0],"sequence":[1,29,45],"length":[2,30,46],"has":[3,76,81],"become":[4],"a":[5,39,82,87,95,103,115,160,169],"critical":[6],"demand":[7],"in":[8,94],"the":[9,27,55,67,72,127,165],"era":[10],"of":[11],"large":[12],"language":[13,145],"models.":[14],"However,":[15],"existing":[16,128],"methods":[17],"struggle":[18],"with":[19,126],"either":[20],"computational":[21],"complexity":[22,85],"or":[23,163],"model":[24],"expressivity,":[25],"rendering":[26],"maximum":[28],"restricted.":[31],"To":[32],"address":[33],"this":[34],"issue,":[35],"we":[36,61],"introduce":[37],"LongNet,":[38],"Transformer":[40],"variant":[41],"that":[42,134],"can":[43,99,122],"scale":[44],"to":[47],"more":[48],"than":[49],"1":[50],"billion":[51],"tokens,":[52],"without":[53],"sacrificing":[54],"performance":[56,138],"on":[57,139],"shorter":[58],"sequences.":[59],"Specifically,":[60],"propose":[62],"dilated":[63,112],"attention,":[64,120],"which":[65,121],"expands":[66],"attentive":[68],"field":[69],"exponentially":[70],"as":[71,102,168],"distance":[73],"grows.":[74],"LongNet":[75,135],"significant":[77],"advantages:":[78],"1)":[79],"it":[80,98],"linear":[83],"computation":[84],"and":[86,143],"logarithm":[88],"dependency":[89],"between":[90],"any":[91],"two":[92],"tokens":[93],"sequence;":[96],"2)":[97],"be":[100,123],"served":[101],"distributed":[104],"trainer":[105],"for":[106,118,153],"extremely":[107],"long":[108,156],"sequences;":[109],"3)":[110],"its":[111],"attention":[113],"is":[114],"drop-in":[116],"replacement":[117],"standard":[119],"seamlessly":[124],"integrated":[125],"Transformer-based":[129],"optimization.":[130],"Experiments":[131],"results":[132],"demonstrate":[133],"yields":[136],"strong":[137],"both":[140],"long-sequence":[141],"modeling":[142,154],"general":[144],"tasks.":[146],"Our":[147],"work":[148],"opens":[149],"up":[150],"new":[151],"possibilities":[152],"very":[155],"sequences,":[157],"e.g.,":[158],"treating":[159],"whole":[161],"corpus":[162],"even":[164],"entire":[166],"Internet":[167],"sequence.":[170]},"counts_by_year":[{"year":2026,"cited_by_count":67},{"year":2025,"cited_by_count":75},{"year":2024,"cited_by_count":88},{"year":2023,"cited_by_count":21}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-07-07T00:00:00"}