{"id":"https://openalex.org/W7163990211","doi":"https://doi.org/10.48550/arxiv.2606.07703","title":"How Much Dense Attention is Necessary? Oracle-Guided Sparse Prefill for Full/GQA Layers in Hybrid Long-Context Models","display_name":"How Much Dense Attention is Necessary? Oracle-Guided Sparse Prefill for Full/GQA Layers in Hybrid Long-Context Models","publication_year":2026,"publication_date":"2026-06-05","ids":{"openalex":"https://openalex.org/W7163990211","doi":"https://doi.org/10.48550/arxiv.2606.07703"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.07703","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07703","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.07703","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100732832","display_name":"Hongxing Wang","orcid":"https://orcid.org/0000-0003-2569-0821"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Hongxing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025770235","display_name":"Harenome Razanajato","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Razanajato, Harenome","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138257390","display_name":"Zhen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138216945","display_name":"Yujie Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Yujie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5119011514","display_name":"Hongsheng Liu","orcid":"https://orcid.org/0000-0003-3011-3893"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Hongsheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3840999901294708,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.3840999901294708,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12292","display_name":"Graph Theory and Algorithms","score":0.12200000137090683,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.06870000064373016,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/oracle","display_name":"Oracle","score":0.6101999878883362},{"id":"https://openalex.org/keywords/row","display_name":"Row","score":0.5400000214576721},{"id":"https://openalex.org/keywords/realization","display_name":"Realization (probability)","score":0.5314000248908997},{"id":"https://openalex.org/keywords/granularity","display_name":"Granularity","score":0.45509999990463257},{"id":"https://openalex.org/keywords/offset","display_name":"Offset (computer science)","score":0.4345000088214874},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.4180999994277954},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.4072999954223633},{"id":"https://openalex.org/keywords/macro","display_name":"Macro","score":0.3955000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7473999857902527},{"id":"https://openalex.org/C55166926","wikidata":"https://www.wikidata.org/wiki/Q2892946","display_name":"Oracle","level":2,"score":0.6101999878883362},{"id":"https://openalex.org/C135598885","wikidata":"https://www.wikidata.org/wiki/Q1366302","display_name":"Row","level":2,"score":0.5400000214576721},{"id":"https://openalex.org/C2781089630","wikidata":"https://www.wikidata.org/wiki/Q21856745","display_name":"Realization (probability)","level":2,"score":0.5314000248908997},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4844000041484833},{"id":"https://openalex.org/C177774035","wikidata":"https://www.wikidata.org/wiki/Q1246948","display_name":"Granularity","level":2,"score":0.45509999990463257},{"id":"https://openalex.org/C175291020","wikidata":"https://www.wikidata.org/wiki/Q1156822","display_name":"Offset (computer science)","level":2,"score":0.4345000088214874},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.4180999994277954},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.4072999954223633},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.3955000042915344},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.36660000681877136},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.36500000953674316},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.3384999930858612},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.32100000977516174},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3149000108242035},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.3140999972820282},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.30790001153945923},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.28610000014305115},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.27079999446868896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.26930001378059387},{"id":"https://openalex.org/C152752567","wikidata":"https://www.wikidata.org/wiki/Q116877","display_name":"Code refactoring","level":3,"score":0.26910001039505005},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.2653000056743622},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2651999890804291},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.26080000400543213}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.07703","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07703","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.07703","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.07703","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Long-context":[0],"prefill":[1],"remains":[2],"expensive":[3],"because":[4],"full/GQA":[5],"layers":[6],"still":[7],"score":[8],"the":[9,98,124,142,152],"historical":[10],"sequence,":[11],"even":[12],"in":[13],"hybrid":[14],"models":[15],"with":[16],"local,":[17],"sparse,":[18],"linear,":[19],"or":[20],"recurrent":[21],"components.":[22],"We":[23,42],"study":[24],"how":[25],"much":[26],"dense":[27,60,137,202],"attention":[28,68],"is":[29,75],"needed":[30],"to":[31,116,236],"preserve":[32],"task-level":[33],"behavior":[34],"under":[35],"explicit":[36],"support":[37,172],"granularity":[38],"and":[39,55,66,83,90,109,149,160,194,227],"top-k":[40,46],"budgets.":[41],"introduce":[43,174],"an":[44],"attention-mass":[45,138],"oracle":[47,74,101,223],"for":[48,52,190,196],"existing":[49],"GQA":[50],"checkpoints:":[51],"each":[53],"layer":[54],"query":[56],"position,":[57],"it":[58],"computes":[59],"attention,":[61],"selects":[62],"head-averaged":[63],"token":[64],"support,":[65],"recomputes":[67],"only":[69],"on":[70,192,198],"that":[71],"support.":[72],"The":[73],"a":[76,80,110,128,175,231],"diagnostic":[77],"reference,":[78],"not":[79,215],"deployable":[81],"accelerator,":[82],"separates":[84,222],"sparse-budget":[85],"feasibility":[86],"from":[87,114,136],"indexer":[88,131],"error":[89],"runtime":[91,228],"realization":[92,177],"effects.":[93],"On":[94],"Qwen-family":[95],"retrieval-heavy":[96],"evaluations,":[97],"longest":[99],"per-query":[100],"rows":[102,208],"stay":[103],"within":[104,119],"1":[105],"point":[106],"of":[107,188],"dense,":[108],"Qwen3.5-9B":[111,150,197],"RULER-style":[112],"sweep":[113],"4K":[115],"100K":[117],"stays":[118],"0.48":[120],"points.":[121],"Guided":[122],"by":[123,133],"oracle,":[125],"we":[126],"derive":[127],"head-collapsed":[129],"auxiliary":[130],"trained":[132],"KL":[134],"distillation":[135],"distributions":[139],"while":[140],"keeping":[141],"backbone":[143],"frozen.":[144],"With":[145],"separately":[146],"distilled":[147],"Qwen3.5-0.8B":[148,191],"indexers,":[151],"reported":[153],"16K/32K":[154],"validation":[155],"macro":[156],"gaps":[157],"are":[158],"+2.04":[159],"+1.13":[161],"points,":[162],"treated":[163],"as":[164],"quality":[165],"preservation":[166],"rather":[167],"than":[168],"improvement;":[169],"fused":[170],"selection-block-shared":[171],"can":[173],"larger":[176],"gap.":[178],"Preliminary":[179],"single-card":[180],"TTFT":[181],"measurements":[182],"show":[183],"distilled-indexer":[184,225],"sparse":[185],"serving":[186],"speedups":[187],"1.71x":[189],"NPU":[193],"1.93x":[195],"GPU":[199],"against":[200],"its":[201],"FlashAttention-2":[203],"baseline.":[204],"Additional":[205],"random-init":[206],"stress":[207],"reach":[209],"3.44x,":[210],"indicating":[211],"sparse-runtime":[212],"headroom":[213],"but":[214],"validated":[216],"output":[217],"quality.":[218],"This":[219],"first":[220],"release":[221],"feasibility,":[224],"quality,":[226],"headroom,":[229],"leaving":[230],"fully":[232],"matched":[233],"quality-latency":[234],"frontier":[235],"future":[237],"work.":[238]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-10T00:00:00"}