{"id":"https://openalex.org/W7152993436","doi":"https://doi.org/10.48550/arxiv.2604.07394","title":"Flux Attention: Context-Aware Hybrid Attention for Efficient LLMs Inference","display_name":"Flux Attention: Context-Aware Hybrid Attention for Efficient LLMs Inference","publication_year":2026,"publication_date":"2026-04-08","ids":{"openalex":"https://openalex.org/W7152993436","doi":"https://doi.org/10.48550/arxiv.2604.07394"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.07394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.07394","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123981501","display_name":"Quantong Qiu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiu, Quantong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123916171","display_name":"Zhiyi Hong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hong, Zhiyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133344707","display_name":"Yi Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133391534","display_name":"Haitian Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Haitian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133343534","display_name":"Kebin Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Kebin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031021503","display_name":"Qingqing Dang","orcid":"https://orcid.org/0000-0001-5483-3778"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dang, Qingqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133366490","display_name":"Juntao Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Juntao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5133388334","display_name":"Min Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Min","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.3463999927043915,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.3463999927043915,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08919999748468399,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.08820000290870667,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.65829998254776},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.598800003528595},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5680000185966492},{"id":"https://openalex.org/keywords/synchronization","display_name":"Synchronization (alternating current)","score":0.5199999809265137},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.439300000667572},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.4345000088214874},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.4034000039100647},{"id":"https://openalex.org/keywords/emulation","display_name":"Emulation","score":0.40139999985694885},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.3668999969959259}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.737500011920929},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.65829998254776},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.598800003528595},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5680000185966492},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.5199999809265137},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.439300000667572},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.4345000088214874},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.4034000039100647},{"id":"https://openalex.org/C149810388","wikidata":"https://www.wikidata.org/wiki/Q5374873","display_name":"Emulation","level":2,"score":0.40139999985694885},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3734000027179718},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.3668999969959259},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3483999967575073},{"id":"https://openalex.org/C71559656","wikidata":"https://www.wikidata.org/wiki/Q671298","display_name":"Divide and conquer algorithms","level":2,"score":0.3422999978065491},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.34209999442100525},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.34040001034736633},{"id":"https://openalex.org/C117896860","wikidata":"https://www.wikidata.org/wiki/Q11376","display_name":"Acceleration","level":2,"score":0.3181000053882599},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.3149000108242035},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.31360000371932983},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.27549999952316284},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2705000042915344},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C157764524","wikidata":"https://www.wikidata.org/wiki/Q1383412","display_name":"Throughput","level":3,"score":0.25859999656677246},{"id":"https://openalex.org/C41661131","wikidata":"https://www.wikidata.org/wiki/Q220764","display_name":"Interrupt","level":3,"score":0.25529998540878296}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.07394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.07394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"quadratic":[1],"computational":[2,60,133],"complexity":[3],"of":[4,50,149,183],"standard":[5],"attention":[6,20,87],"mechanisms":[7,21],"presents":[8],"a":[9,31,81,95,140,168],"severe":[10,59],"scalability":[11],"bottleneck":[12],"for":[13],"LLMs":[14],"in":[15,189],"long-context":[16,158],"scenarios.":[17],"While":[18],"hybrid":[19],"combining":[22],"Full":[23],"Attention":[24,28,166],"(FA)":[25],"and":[26,63,159,173,187,192],"Sparse":[27],"(SA)":[29],"offer":[30],"potential":[32],"solution,":[33],"existing":[34],"methods":[35],"typically":[36],"rely":[37],"on":[38,115,151],"static":[39],"allocation":[40],"ratios":[41],"that":[42,84,164],"fail":[43],"to":[44,110,185],"accommodate":[45],"the":[46,90,103,116,190],"variable":[47],"retrieval":[48,125],"demands":[49],"different":[51],"tasks.":[52],"Furthermore,":[53],"head-level":[54],"dynamic":[55],"sparsity":[56],"often":[57],"introduces":[58],"load":[61],"imbalance":[62],"synchronization":[64],"long-tails,":[65],"which":[66],"hinder":[67],"hardware":[68],"acceleration":[69],"during":[70],"autoregressive":[71],"decoding.":[72],"To":[73],"bridge":[74],"this":[75],"gap,":[76],"we":[77],"introduce":[78],"Flux":[79,165],"Attention,":[80],"context-aware":[82],"framework":[83,144],"dynamically":[85],"optimizes":[86],"computation":[88],"at":[89],"layer":[91,109],"level.":[92],"By":[93],"integrating":[94],"lightweight":[96],"Layer":[97],"Router":[98],"into":[99,135],"frozen":[100],"pretrained":[101],"LLMs,":[102],"proposed":[104],"method":[105],"adaptively":[106],"routes":[107],"each":[108],"FA":[111],"or":[112],"SA":[113],"based":[114],"input":[117],"context.":[118],"This":[119],"layer-wise":[120],"routing":[121],"preserves":[122],"high-fidelity":[123],"information":[124],"while":[126],"ensuring":[127],"contiguous":[128],"memory":[129],"access,":[130],"translating":[131],"theoretical":[132],"reductions":[134],"practical":[136],"wall-clock":[137],"speedups.":[138],"As":[139],"parameter-efficient":[141],"approach,":[142],"our":[143],"requires":[145],"only":[146],"12":[147],"hours":[148],"training":[150],"8$\\times$A800":[152],"GPUs.":[153],"Extensive":[154],"experiments":[155],"across":[156],"multiple":[157],"mathematical":[160],"reasoning":[161],"benchmarks":[162],"demonstrate":[163],"achieves":[167],"superior":[169],"trade-off":[170],"between":[171],"performance":[172],"inference":[174],"speed":[175,181],"compared":[176],"with":[177,180],"baseline":[178],"models,":[179],"improvements":[182],"up":[184],"$2.8\\times$":[186],"$2.0\\times$":[188],"prefill":[191],"decode":[193],"stages.":[194]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-11T00:00:00"}