{"id":"https://openalex.org/W4393147252","doi":"https://doi.org/10.1609/aaai.v38i17.29855","title":"Exploring Transformer Extrapolation","display_name":"Exploring Transformer Extrapolation","publication_year":2024,"publication_date":"2024-03-24","ids":{"openalex":"https://openalex.org/W4393147252","doi":"https://doi.org/10.1609/aaai.v38i17.29855"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v38i17.29855","doi":"https://doi.org/10.1609/aaai.v38i17.29855","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1609/aaai.v38i17.29855","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29855/31489","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29855/31489","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100742403","display_name":"Zhen Qin","orcid":"https://orcid.org/0009-0000-0388-8840","observed_orcids":["https://orcid.org/0009-0000-0388-8840"]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Qin","raw_affiliation_strings":["OpenNLPLab, Shanghai AI Lab\nTapTap"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenNLPLab, Shanghai AI Lab\nTapTap","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066621365","display_name":"Yiran Zhong","orcid":"https://orcid.org/0000-0003-1404-3610","observed_orcids":["https://orcid.org/0000-0003-1404-3610"]},"institutions":[{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiran Zhong","raw_affiliation_strings":["OpenNLPLab, Shanghai AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenNLPLab, Shanghai AI Lab","institution_ids":["https://openalex.org/I4391012619"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111614248","display_name":"Hui Deng","orcid":"https://orcid.org/0000-0002-7116-7188","observed_orcids":["https://orcid.org/0000-0002-7116-7188","https://orcid.org/0000-0002-8002-236X","https://orcid.org/0009-0003-5135-9590","https://orcid.org/0009-0006-7611-7485"]},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hui Deng","raw_affiliation_strings":["Northwestern Polytechnical University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern Polytechnical University","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6108,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.68601491,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":96},"biblio":{"volume":"38","issue":"17","first_page":"18897","last_page":"18905"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9160000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9160000085830688,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05490000173449516,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.00279999990016222,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/length-extrapolation","display_name":"length extrapolation","score":0.925000011920929},{"id":"https://openalex.org/keywords/relative-position-encoding","display_name":"relative position encoding","score":0.8989999890327454},{"id":"https://openalex.org/keywords/exponential-convergence","display_name":"exponential convergence","score":0.7229999899864197},{"id":"https://openalex.org/keywords/language-modeling","display_name":"language modeling","score":0.6970000267028809}],"concepts":[{"id":"https://openalex.org/C132459708","wikidata":"https://www.wikidata.org/wiki/Q744069","display_name":"Extrapolation","level":2,"score":0.6761149168014526},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.47366079688072205},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.20023155212402344},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19858315587043762},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.182689368724823},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1147109866142273},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.10042420029640198}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v38i17.29855","doi":"https://doi.org/10.1609/aaai.v38i17.29855","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1609/aaai.v38i17.29855","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29855/31489","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2307.10156","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.10156","pdf_url":"https://arxiv.org/pdf/2307.10156","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/29855","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/29855","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2307.10156","doi":"https://doi.org/10.48550/arxiv.2307.10156","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2307.10156","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v38i17.29855","doi":"https://doi.org/10.1609/aaai.v38i17.29855","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1609/aaai.v38i17.29855","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/29855/31489","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393147252.pdf","grobid_xml":"https://content.openalex.org/works/W4393147252.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W1832693441","https://openalex.org/W2064675550","https://openalex.org/W2157331557","https://openalex.org/W2525332836","https://openalex.org/W2526050071","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2896457183","https://openalex.org/W2933138175","https://openalex.org/W2952509486","https://openalex.org/W2965373594","https://openalex.org/W3015468748","https://openalex.org/W3094502228","https://openalex.org/W3097777922","https://openalex.org/W3118781290","https://openalex.org/W3163721282","https://openalex.org/W3196318247","https://openalex.org/W3196974791","https://openalex.org/W3214370733","https://openalex.org/W4221153789","https://openalex.org/W4225525539","https://openalex.org/W4281381535","https://openalex.org/W4288804157","https://openalex.org/W4306815714","https://openalex.org/W4375959361","https://openalex.org/W4380520356","https://openalex.org/W4384811648","https://openalex.org/W4385374362","https://openalex.org/W4385570109","https://openalex.org/W4385573415","https://openalex.org/W4386566638","https://openalex.org/W4388979610","https://openalex.org/W4389523728","https://openalex.org/W4393159092","https://openalex.org/W6788175385","https://openalex.org/W6815833287","https://openalex.org/W7133201027"],"related_works":["https://openalex.org/W4404782324","https://openalex.org/W4312091893","https://openalex.org/W4312091411","https://openalex.org/W4416035361","https://openalex.org/W6903487831","https://openalex.org/W7005554804","https://openalex.org/W4414893801","https://openalex.org/W7133185274","https://openalex.org/W4417308017","https://openalex.org/W4415914879"],"abstract_inverted_index":{"Length":[0],"extrapolation":[1,54,72],"has":[2,24],"attracted":[3],"considerable":[4],"attention":[5],"recently":[6],"since":[7],"it":[8],"allows":[9],"transformers":[10],"to":[11,57,63,87,98,135,159,171],"be":[12,30,58],"tested":[13],"on":[14,45,116,151,183],"longer":[15],"sequences":[16],"than":[17],"those":[18],"used":[19],"in":[20,112],"training.":[21],"Previous":[22],"research":[23],"shown":[25],"that":[26,82,96],"this":[27,89],"property":[28,90],"can":[29],"attained":[31],"by":[32],"using":[33],"carefully":[34],"designed":[35],"Relative":[36],"Positional":[37],"Encodings":[38],"(RPEs).":[39],"While":[40],"these":[41,184],"methods":[42],"perform":[43],"well":[44],"a":[46,74,83,117,122,129],"variety":[47,118],"of":[48,67,119,140,163],"corpora,":[49],"the":[50,94,99,108,125,137,152,161],"conditions":[51,109],"for":[52,70],"length":[53,71],"have":[55],"yet":[56],"investigated.":[59],"This":[60],"paper":[61],"attempts":[62],"determine":[64],"what":[65],"types":[66],"RPEs":[68,141],"allow":[69],"through":[73],"thorough":[75],"mathematical":[76],"and":[77,110,156],"empirical":[78],"analysis.":[79],"We":[80,167],"discover":[81],"transformer":[84],"is":[85,187],"certain":[86],"possess":[88],"as":[91,93],"long":[92],"series":[95],"corresponds":[97],"RPE's":[100],"exponential":[101],"converges.":[102],"Two":[103],"practices":[104],"are":[105,149],"derived":[106],"from":[107,124],"examined":[111],"language":[113],"modeling":[114],"tasks":[115],"corpora.":[120],"As":[121],"bonus":[123],"conditions,":[126],"we":[127],"derive":[128],"new":[130],"Theoretical":[131],"Receptive":[132,173],"Field":[133,174],"(TRF)":[134],"measure":[136],"receptive":[138],"field":[139],"without":[142],"taking":[143],"any":[144],"training":[145],"steps.":[146],"Extensive":[147],"experiments":[148],"conducted":[150],"Wikitext-103,":[153],"Books,":[154],"Github,":[155],"WikiBook":[157],"datasets":[158],"demonstrate":[160],"viability":[162],"our":[164],"discovered":[165],"conditions.":[166],"also":[168],"compare":[169],"TRF":[170],"Empirical":[172],"(ERF)":[175],"across":[176],"different":[177],"models,":[178],"showing":[179],"consistently":[180],"matched":[181],"trends":[182],"datasets.":[185],"Code":[186],"released":[188],"at:":[189],"https://github.com/OpenNLPLab/Rpe.":[190]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-03-26T00:00:00"}