{"id":"https://openalex.org/W7151232163","doi":"https://doi.org/10.48550/arxiv.2604.03260","title":"Why Attend to Everything? Focus is the Key","display_name":"Why Attend to Everything? Focus is the Key","publication_year":2026,"publication_date":"2026-03-12","ids":{"openalex":"https://openalex.org/W7151232163","doi":"https://doi.org/10.48550/arxiv.2604.03260"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.03260","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5050876115","display_name":"Hengshuai Yao","orcid":"https://orcid.org/0000-0003-1258-1845"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Hengshuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133109572","display_name":"Xing Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130090677","display_name":"Ahmed Murtadha","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Murtadha, Ahmed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133084917","display_name":"Jin Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yadkori, Yasin Abbasi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yadkori, Yasin Abbasi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133123053","display_name":"Shuai Shao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shao, Shuai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Changling","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Changling","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133125015","display_name":"Guan Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Guan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133065945","display_name":"Mingli Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Mingli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133124740","display_name":"William C. Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, William","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Song, Sen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Song, Sen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1858000010251999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.1858000010251999,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.16060000658035278,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.08910000324249268,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perplexity","display_name":"Perplexity","score":0.8737999796867371},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.734000027179718},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.7200000286102295},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6604999899864197},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5849999785423279},{"id":"https://openalex.org/keywords/centroid","display_name":"Centroid","score":0.4934999942779541},{"id":"https://openalex.org/keywords/focus-group","display_name":"Focus group","score":0.38850000500679016}],"concepts":[{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.8737999796867371},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7351999878883362},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.734000027179718},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.7200000286102295},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6604999899864197},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5849999785423279},{"id":"https://openalex.org/C146599234","wikidata":"https://www.wikidata.org/wiki/Q511093","display_name":"Centroid","level":2,"score":0.4934999942779541},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43650001287460327},{"id":"https://openalex.org/C56995899","wikidata":"https://www.wikidata.org/wiki/Q1126687","display_name":"Focus group","level":2,"score":0.38850000500679016},{"id":"https://openalex.org/C136886441","wikidata":"https://www.wikidata.org/wiki/Q926129","display_name":"Normalization (sociology)","level":2,"score":0.3853999972343445},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.3702999949455261},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3675999939441681},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.34940001368522644},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33219999074935913},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.31630000472068787},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.31119999289512634},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3109999895095825},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29490000009536743},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.27059999108314514}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.03260","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.03260","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5504781603813171}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Standard":[0],"attention":[1,8,118,123,126,137],"scales":[2],"quadratically":[3],"with":[4,159],"sequence":[5],"length.":[6],"Efficient":[7],"methods":[9],"reduce":[10],"this":[11],"O(n^2)":[12],"cost,":[13],"but":[14],"when":[15,138],"retrofitted":[16],"into":[17],"pretrained":[18,81,100,165],"models,":[19],"they":[20],"often":[21],"degrade":[22],"perplexity,":[23],"downstream":[24,106],"accuracy,":[25],"or":[26],"both.":[27],"We":[28],"introduce":[29],"Focus,":[30],"a":[31,41,156],"method":[32],"that":[33,96],"learns":[34],"which":[35],"token":[36,57],"pairs":[37,58],"matter.":[38],"Focus":[39,72,98,122,148,171],"adds":[40],"small":[42],"set":[43],"of":[44],"learnable":[45],"centroids--as":[46],"few":[47],"as":[48,54],"148K":[49],"parameters":[50,115],"per":[51],"layer--that":[52],"act":[53],"gates:":[55],"only":[56,85],"belonging":[59],"to":[60,66,79,113],"the":[61,86,163],"same":[62],"centroid":[63],"group":[64,153],"attend":[65],"each":[67],"other":[68],"over":[69],"long":[70],"ranges.":[71],"is":[73,149],"composable:":[74],"it":[75],"can":[76],"be":[77],"added":[78],"any":[80],"model":[82,109],"by":[83],"training":[84],"centroids":[87],"while":[88],"keeping":[89],"all":[90],"original":[91,164],"weights":[92],"frozen.":[93],"Experiments":[94],"show":[95],"composing":[97],"onto":[99],"models":[101],"yields":[102],"zero":[103],"degradation":[104],"on":[105],"benchmarks":[107],"across":[108],"sizes":[110],"from":[111,140],"124M":[112,128],"70B":[114],"and":[116,134],"five":[117],"architectures.":[119],"Surprisingly,":[120],"sparse":[121],"outperforms":[124],"full":[125,136],"at":[127,142,176],"scale":[129,144],"(30.3":[130],"vs.":[131,146],"31.4":[132],"perplexity)":[133],"matches":[135],"trained":[139],"scratch":[141],"7B":[143],"(13.82":[145],"13.89).":[147],"also":[150],"fast:":[151],"top-k":[152],"membership":[154],"gives":[155],"2x":[157],"speedup":[158,175],"better":[160],"quality":[161],"than":[162],"model.":[166],"Using":[167],"our":[168],"FlashAttention":[169],"decomposition,":[170],"achieves":[172],"an":[173],"8.6x":[174],"1M":[177],"tokens":[178],"without":[179],"custom":[180],"kernels.":[181]},"counts_by_year":[],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2026-04-08T00:00:00"}