{"id":"https://openalex.org/W2964110616","doi":"https://doi.org/10.18653/v1/p19-1285","title":"Transformer-XL: Attentive Language Models beyond a Fixed-Length Context","display_name":"Transformer-XL: Attentive Language Models beyond a Fixed-Length Context","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2964110616","doi":"https://doi.org/10.18653/v1/p19-1285","mag":"2964110616"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1285","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1285","pdf_url":"https://www.aclweb.org/anthology/P19-1285.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1285.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091869105","display_name":"Zihang Dai","orcid":"https://orcid.org/0000-0002-6509-5640"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zihang Dai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050303775","display_name":"Zhilin Yang","orcid":"https://orcid.org/0000-0003-0944-4903"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhilin Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106542734","display_name":"Yiming Yang","orcid":"https://orcid.org/0009-0006-3569-0023"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yiming Yang","raw_affiliation_strings":["Carnegie Mellon University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109861718","display_name":"Jaime Carbonell","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaime Carbonell","raw_affiliation_strings":["Carnegie Mellon University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088551093","display_name":"Quoc V. Le","orcid":"https://orcid.org/0000-0002-1087-2844"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Quoc Le","raw_affiliation_strings":["Google Brain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Brain","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071983998","display_name":"Ruslan Salakhutdinov","orcid":"https://orcid.org/0000-0002-3752-2756"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruslan Salakhutdinov","raw_affiliation_strings":["Carnegie Mellon University,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University,","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3168,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2978","last_page":"2988"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.9797999858856201,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.7861325740814209},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7611839771270752},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.7015364170074463},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6623331308364868},{"id":"https://openalex.org/keywords/treebank","display_name":"Treebank","score":0.5795519351959229},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5598369836807251},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.5235882997512817},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4210675060749054},{"id":"https://openalex.org/keywords/dependency","display_name":"Dependency (UML)","score":0.2742655873298645},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.09777900576591492},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.08331140875816345}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.7861325740814209},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7611839771270752},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.7015364170074463},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6623331308364868},{"id":"https://openalex.org/C206134035","wikidata":"https://www.wikidata.org/wiki/Q811525","display_name":"Treebank","level":3,"score":0.5795519351959229},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5598369836807251},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5235882997512817},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4210675060749054},{"id":"https://openalex.org/C19768560","wikidata":"https://www.wikidata.org/wiki/Q320727","display_name":"Dependency (UML)","level":2,"score":0.2742655873298645},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.09777900576591492},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.08331140875816345},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1285","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1285","pdf_url":"https://www.aclweb.org/anthology/P19-1285.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1285","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1285","pdf_url":"https://www.aclweb.org/anthology/P19-1285.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7699999809265137}],"awards":[{"id":"https://openalex.org/G2581707695","display_name":"BIGDATA: F: Large-Scale Transductive Learning from Heterogeneous Data Sources","funder_award_id":"1546329","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3550935145","display_name":null,"funder_award_id":"N000141812861","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G5420615920","display_name":null,"funder_award_id":"IIS-1546329","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G946752494","display_name":null,"funder_award_id":"IIS1763562","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320309480","display_name":"Nvidia","ror":"https://ror.org/03jdj4y14"},{"id":"https://openalex.org/F4320332359","display_name":"Office of Science","ror":"https://ror.org/00mmn6b08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320337506","display_name":"Advanced Scientific Computing Research","ror":"https://ror.org/0012c7r22"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2964110616.pdf","grobid_xml":"https://content.openalex.org/works/W2964110616.grobid-xml"},"referenced_works_count":80,"referenced_works":["https://openalex.org/W179875071","https://openalex.org/W1800356822","https://openalex.org/W1810943226","https://openalex.org/W1899504021","https://openalex.org/W1999965501","https://openalex.org/W2064675550","https://openalex.org/W2133564696","https://openalex.org/W2140679639","https://openalex.org/W2145543707","https://openalex.org/W2170973209","https://openalex.org/W2197913429","https://openalex.org/W2207587218","https://openalex.org/W2212703438","https://openalex.org/W2259472270","https://openalex.org/W2510842514","https://openalex.org/W2519314406","https://openalex.org/W2525246036","https://openalex.org/W2525332836","https://openalex.org/W2549416390","https://openalex.org/W2549476280","https://openalex.org/W2553303224","https://openalex.org/W2567070169","https://openalex.org/W2605203995","https://openalex.org/W2611669587","https://openalex.org/W2618854269","https://openalex.org/W2743945814","https://openalex.org/W2778817245","https://openalex.org/W2792376130","https://openalex.org/W2792764867","https://openalex.org/W2793273050","https://openalex.org/W2795285343","https://openalex.org/W2804845563","https://openalex.org/W2810075754","https://openalex.org/W2891369367","https://openalex.org/W2891815651","https://openalex.org/W2896457183","https://openalex.org/W2900096133","https://openalex.org/W2950527759","https://openalex.org/W2951104886","https://openalex.org/W2951672049","https://openalex.org/W2951714314","https://openalex.org/W2952339051","https://openalex.org/W2952723479","https://openalex.org/W2952729433","https://openalex.org/W2962739339","https://openalex.org/W2962746461","https://openalex.org/W2962754271","https://openalex.org/W2962964385","https://openalex.org/W2963034893","https://openalex.org/W2963088785","https://openalex.org/W2963266340","https://openalex.org/W2963304263","https://openalex.org/W2963341956","https://openalex.org/W2963351145","https://openalex.org/W2963403868","https://openalex.org/W2963537482","https://openalex.org/W2963573053","https://openalex.org/W2963631907","https://openalex.org/W2963735467","https://openalex.org/W2963925437","https://openalex.org/W2963951265","https://openalex.org/W2963970792","https://openalex.org/W2963983719","https://openalex.org/W2964019776","https://openalex.org/W2964059481","https://openalex.org/W2964269252","https://openalex.org/W2964308564","https://openalex.org/W2964347220","https://openalex.org/W2964348070","https://openalex.org/W4285719527","https://openalex.org/W4293718192","https://openalex.org/W4294555862","https://openalex.org/W4297788867","https://openalex.org/W4298422451","https://openalex.org/W4299838440","https://openalex.org/W4300687381","https://openalex.org/W4302375066","https://openalex.org/W4303633609","https://openalex.org/W4385245566","https://openalex.org/W4394642966"],"related_works":["https://openalex.org/W4301800915","https://openalex.org/W2739034105","https://openalex.org/W2963839582","https://openalex.org/W2964047924","https://openalex.org/W2743945814","https://openalex.org/W2962832505","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W3049463507","https://openalex.org/W3150294986"],"abstract_inverted_index":{"Transformers":[0,104],"have":[1],"a":[2,12,23,33,43,48,69],"potential":[3],"of":[4,18,42,113,153],"learning":[5,30],"longer-term":[6,59],"dependency,":[7,60],"but":[8,61],"are":[9,161],"limited":[10],"by":[11],"fixed-length":[13],"context":[14,65],"in":[15,163],"the":[16,64,110],"setting":[17],"language":[19],"modeling.":[20],"We":[21],"propose":[22],"novel":[24,49,148],"neural":[25],"architecture":[26],"Transformer-XL":[27,71,142],"that":[28,74],"enables":[29,57],"dependency":[31,73],"beyond":[32],"fixed":[34],"length":[35],"without":[36],"disrupting":[37],"temporal":[38],"coherence.":[39],"It":[40],"consists":[41],"segment-level":[44],"recurrence":[45],"mechanism":[46],"and":[47,80,92,95,130,159,166],"positional":[50],"encoding":[51],"scheme.":[52],"Our":[53,155],"method":[54],"not":[55],"only":[56,139],"capturing":[58],"also":[62],"resolves":[63],"fragmentation":[66],"problem.":[67],"As":[68],"result,":[70],"learns":[72],"is":[75,96],"80%":[76],"longer":[77,82],"than":[78,83,102],"RNNs":[79],"450%":[81],"vanilla":[84,103],"Transformers,":[85],"achieves":[86],"better":[87],"performance":[88],"on":[89,117,120,123,126,132,140],"both":[90,164],"short":[91],"long":[93],"sequences,":[94],"up":[97],"to":[98,115,144],"1,800+":[99],"times":[100],"faster":[101],"during":[105],"evaluation.":[106],"Notably,":[107],"we":[108],"improve":[109],"state-ofthe-art":[111],"results":[112],"bpc/perplexity":[114],"0.99":[116],"en-wiki8,":[118],"1.08":[119],"text8,":[121],"18.3":[122],"WikiText-103,":[124,141],"21.8":[125],"One":[127],"Billion":[128],"Word,":[129],"54.5":[131],"Penn":[133],"Treebank":[134],"(without":[135],"finetuning).":[136],"When":[137],"trained":[138],"manages":[143],"generate":[145],"reasonably":[146],"coherent,":[147],"text":[149],"articles":[150],"with":[151],"thousands":[152],"tokens.":[154],"code,":[156],"pretrained":[157],"models,":[158],"hyperparameters":[160],"available":[162],"Tensorflow":[165],"PyTorch":[167],"1":[168],".":[169]},"counts_by_year":[{"year":2026,"cited_by_count":85},{"year":2025,"cited_by_count":341},{"year":2024,"cited_by_count":422},{"year":2023,"cited_by_count":570},{"year":2022,"cited_by_count":492},{"year":2021,"cited_by_count":659},{"year":2020,"cited_by_count":464},{"year":2019,"cited_by_count":134},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-18T07:39:51.176621","created_date":"2025-10-10T00:00:00"}