{"id":"https://openalex.org/W7161580318","doi":"https://doi.org/10.48550/arxiv.2605.15514","title":"RoPE Distinguishes Neither Positions Nor Tokens in Long Contexts, Provably","display_name":"RoPE Distinguishes Neither Positions Nor Tokens in Long Contexts, Provably","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161580318","doi":"https://doi.org/10.48550/arxiv.2605.15514"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.15514","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15514","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.15514","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136409294","display_name":"Yufeng Du","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Yufeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136383769","display_name":"Phillip Harris","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Harris, Phillip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045793208","display_name":"Minyang Tian","orcid":"https://orcid.org/0000-0001-8009-9116"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Minyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059877418","display_name":"E. A. Huerta","orcid":"https://orcid.org/0000-0002-9682-3604"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huerta, Eliu A","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004616142","display_name":"Srikanth Ronanki","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ronanki, Srikanth","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009792867","display_name":"Subendhu Rongali","orcid":"https://orcid.org/0000-0003-1000-6118"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rongali, Subendhu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101715504","display_name":"Aram Galstyan","orcid":"https://orcid.org/0000-0003-4215-0886"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Galstyan, Aram","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136355610","display_name":"Hao Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.39259999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.39259999990463257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18629999458789825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.1404999941587448,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.8773999810218811},{"id":"https://openalex.org/keywords/rope","display_name":"Rope","score":0.7383999824523926},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5058000087738037},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.5009999871253967},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.43869999051094055},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.43700000643730164}],"concepts":[{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.8773999810218811},{"id":"https://openalex.org/C162269090","wikidata":"https://www.wikidata.org/wiki/Q1156047","display_name":"Rope","level":2,"score":0.7383999824523926},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6284000277519226},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5058000087738037},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.5009999871253967},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.43869999051094055},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.43700000643730164},{"id":"https://openalex.org/C2779808786","wikidata":"https://www.wikidata.org/wiki/Q6664603","display_name":"Locality","level":2,"score":0.3828999996185303},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.36629998683929443},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3637999892234802},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35670000314712524},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.35510000586509705},{"id":"https://openalex.org/C182306322","wikidata":"https://www.wikidata.org/wiki/Q1779371","display_name":"Order (exchange)","level":2,"score":0.3091000020503998},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.2922999858856201},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2840000092983246},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.26969999074935913}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.15514","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15514","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.15514","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.15514","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,32,115],"identify":[1],"intrinsic":[2],"limitations":[3],"of":[4,23,106],"Rotary":[5],"Positional":[6],"Embeddings":[7],"(RoPE)":[8],"in":[9,76,177,223],"Transformer-based":[10],"long-context":[11,179,226],"language":[12,227],"models.":[13,228],"Our":[14,193,207],"theoretical":[15],"analysis":[16,195],"abstracts":[17],"away":[18],"from":[19],"the":[20,24,104,119,151,166,170,188],"specific":[21],"content":[22],"context":[25,36],"and":[26,43,217],"depends":[27],"only":[28],"on":[29],"its":[30,51,56],"length.":[31],"prove":[33,117],"that":[34,47,82,118,197,210],"as":[35],"length":[37],"increases,":[38],"RoPE-based":[39],"attention":[40,86,120],"becomes":[41],"unpredictable":[42],"loses":[44,55,74],"two":[45],"properties":[46],"are":[48,201],"central":[49],"to":[50,64,131,145,190,203],"effectiveness.":[52],"First,":[53],"it":[54,73],"locality":[57],"bias:":[58],"RoPE":[59,152,171],"is":[60,129],"no":[61,110],"more":[62],"likely":[63],"favor":[65],"nearer":[66],"positions":[67,147,157],"than":[68,88,112],"substantially":[69],"farther":[70],"ones.":[71],"Second,":[72],"consistency":[75],"token":[77,128,218],"relevance:":[78],"a":[79,84,96,126,132,139,143,174],"key":[80,127],"vector":[81],"receives":[83],"higher":[85],"score":[87,98,121],"an":[89],"alternative":[90],"at":[91,99,165],"one":[92],"position":[93,216],"may":[94,220],"receive":[95],"lower":[97],"another.":[100],"In":[101],"both":[102,164],"cases,":[103],"probability":[105],"failure":[107,144],"approaches":[108],"0.5,":[109],"better":[111],"random":[113],"guessing.":[114],"further":[116],"can":[122],"remain":[123],"unchanged":[124],"when":[125],"moved":[130],"different":[133,140,183],"position,":[134],"or":[135,148],"even":[136],"replaced":[137],"by":[138],"token,":[141],"indicating":[142],"distinguish":[146,182,191],"tokens.":[149],"Adjusting":[150],"base":[153,172],"trades":[154],"off":[155],"distinguishing":[156,159],"against":[158],"tokens":[160],"but":[161,185],"cannot":[162],"preserve":[163],"same":[167],"time.":[168],"Increasing":[169],"hyperparameter,":[173],"common":[175],"practice":[176],"today's":[178],"models,":[180],"helps":[181],"tokens,":[184],"inevitably":[186],"sacrifices":[187],"ability":[189],"positions.":[192],"empirical":[194],"shows":[196],"multi-head,":[198],"multi-layer":[199],"architectures":[200],"insufficient":[202],"overcome":[204],"these":[205],"limitations.":[206],"findings":[208],"suggest":[209],"fundamentally":[211],"new":[212],"mechanisms":[213],"for":[214],"encoding":[215],"order":[219],"be":[221],"needed":[222],"future":[224],"Transformer":[225]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-19T00:00:00"}