{"id":"https://openalex.org/W7152032360","doi":"https://doi.org/10.48550/arxiv.2604.06070","title":"Short Data, Long Context: Distilling Positional Knowledge in Transformers","display_name":"Short Data, Long Context: Distilling Positional Knowledge in Transformers","publication_year":2026,"publication_date":"2026-04-07","ids":{"openalex":"https://openalex.org/W7152032360","doi":"https://doi.org/10.48550/arxiv.2604.06070"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.06070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.06070","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5133172193","display_name":"Patrick Huber","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huber, Patrick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133232943","display_name":"Ernie Chang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Ernie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039984997","display_name":"Chinnadhurai Sankar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sankar, Chinnadhurai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133169915","display_name":"Rylan Conway","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Conway, Rylan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045083632","display_name":"I. A. Fedorov","orcid":"https://orcid.org/0000-0002-6209-9830"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fedorov, Igor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026961254","display_name":"Md Rifat Arefin","orcid":"https://orcid.org/0000-0002-6433-6764"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arefin, Md Rifat","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5113432231","display_name":"Adithya Sagar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sagar, Adithya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4611999988555908,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4611999988555908,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.18880000710487366,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.06040000170469284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6412000060081482},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.6025000214576721},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4514000117778778},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.43700000643730164},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.36820000410079956},{"id":"https://openalex.org/keywords/overhead","display_name":"Overhead (engineering)","score":0.34619998931884766},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.3375999927520752}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7062000036239624},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6412000060081482},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.6025000214576721},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4514000117778778},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4415000081062317},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.43700000643730164},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.36820000410079956},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3675999939441681},{"id":"https://openalex.org/C2779960059","wikidata":"https://www.wikidata.org/wiki/Q7113681","display_name":"Overhead (engineering)","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.3375999927520752},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.2928999960422516},{"id":"https://openalex.org/C21200559","wikidata":"https://www.wikidata.org/wiki/Q7451068","display_name":"Sensitivity (control systems)","level":2,"score":0.2906000018119812},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29030001163482666},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.289000004529953},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.27730000019073486},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.26330000162124634},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C2778751112","wikidata":"https://www.wikidata.org/wiki/Q835016","display_name":"Window (computing)","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.06070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.06070","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.06070","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7351509928703308,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Extending":[0],"the":[1,59,93,125,148,155,160,171],"context":[2],"window":[3],"of":[4,61,127],"language":[5],"models":[6,37],"typically":[7],"requires":[8],"expensive":[9],"long-context":[10,29,52,95,175,185],"pre-training,":[11],"posing":[12],"significant":[13],"challenges":[14],"for":[15],"both":[16],"training":[17,44,89],"efficiency":[18],"and":[19,66,132],"data":[20],"collection.":[21],"In":[22],"this":[23],"paper,":[24],"we":[25,76,102,123],"present":[26],"evidence":[27],"that":[28,78,104,143],"retrieval":[30],"capabilities":[31],"can":[32,108],"be":[33],"transferred":[34],"to":[35,139,184],"student":[36,161],"through":[38,58,135],"logit-based":[39,105],"knowledge":[40,98,106],"distillation,":[41],"even":[42],"when":[43],"exclusively":[45],"on":[46],"packed":[47,119],"short-context":[48],"samples":[49],"within":[50],"a":[51],"window.":[53],"We":[54],"provide":[55],"comprehensive":[56],"insights":[57],"lens":[60],"Rotary":[62],"Position":[63],"Embedding":[64],"(RoPE)":[65],"establish":[67],"three":[68],"key":[69,133],"findings.":[70],"First,":[71],"consistent":[72],"with":[73,118,177],"prior":[74],"work,":[75],"show":[77],"phase-wise":[79],"RoPE":[80],"scaling,":[81],"which":[82],"maximizes":[83],"rotational":[84],"spectrum":[85],"utilization":[86],"at":[87],"each":[88],"stage,":[90],"also":[91],"achieves":[92],"best":[94],"performance":[96],"in":[97,153,170],"distillation":[99,107,156],"setups.":[100],"Second,":[101],"demonstrate":[103],"directly":[109],"enable":[110],"positional":[111,128,144],"information":[112,145],"transfer.":[113],"Using":[114],"an":[115],"experimental":[116],"setup":[117],"repeated":[120],"token":[121],"sequences,":[122],"trace":[124],"propagation":[126],"perturbations":[129],"from":[130],"query":[131,172],"vectors":[134],"successive":[136],"transformer":[137],"layers":[138],"output":[140,150],"logits,":[141],"revealing":[142],"systematically":[146],"influences":[147],"teacher's":[149],"distribution":[151],"and,":[152],"turn,":[154],"signal":[157],"received":[158],"by":[159],"model.":[162],"Third,":[163],"our":[164],"analysis":[165],"uncovers":[166],"structured":[167],"update":[168],"patterns":[169],"state":[173],"during":[174],"extension,":[176],"distinct":[178],"parameter":[179],"spans":[180],"exhibiting":[181],"strong":[182],"sensitivity":[183],"training.":[186]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-09T00:00:00"}