{"id":"https://openalex.org/W7165405163","doi":"https://doi.org/10.48550/arxiv.2606.20005","title":"StreamKL: Fast and Memory-Efficient KL Divergence for Boosting Attention Distillation","display_name":"StreamKL: Fast and Memory-Efficient KL Divergence for Boosting Attention Distillation","publication_year":2026,"publication_date":"2026-06-18","ids":{"openalex":"https://openalex.org/W7165405163","doi":"https://doi.org/10.48550/arxiv.2606.20005"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.20005","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20005","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.20005","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055139643","display_name":"Guangda Liu","orcid":"https://orcid.org/0000-0001-8905-7641"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Guangda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051449178","display_name":"Yiquan Wang","orcid":"https://orcid.org/0000-0003-1417-5752"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yiquan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138990674","display_name":"Chengwei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Chengwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139018319","display_name":"Wenhao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wenhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139017426","display_name":"Jing Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Jing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019176355","display_name":"Yiwu Yao","orcid":"https://orcid.org/0000-0003-4642-5840"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Yiwu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123893914","display_name":"Danning Ke","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ke, Danning","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139019117","display_name":"Wenchao Ding","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ding, Wenchao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139008278","display_name":"Jieru Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Jieru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5034000277519226,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.5034000277519226,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.11299999803304672,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.0738999992609024,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.6822999715805054},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.6305000185966492},{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.6079000234603882},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5386000275611877},{"id":"https://openalex.org/keywords/distillation","display_name":"Distillation","score":0.5213000178337097},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.5144000053405762}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7361999750137329},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.6822999715805054},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.6305000185966492},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.6079000234603882},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5386000275611877},{"id":"https://openalex.org/C204030448","wikidata":"https://www.wikidata.org/wiki/Q101017","display_name":"Distillation","level":2,"score":0.5213000178337097},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.5144000053405762},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.44279998540878296},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40799999237060547},{"id":"https://openalex.org/C190839683","wikidata":"https://www.wikidata.org/wiki/Q2448197","display_name":"Train","level":2,"score":0.3944000005722046},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3628000020980835},{"id":"https://openalex.org/C81845259","wikidata":"https://www.wikidata.org/wiki/Q290117","display_name":"Quadratic programming","level":2,"score":0.3499000072479248},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.31859999895095825},{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2800999879837036},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.20005","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20005","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.20005","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20005","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Attention":[0],"distillation,":[1,21],"which":[2],"trains":[3],"one":[4],"attention":[5,35,64,103,151],"distribution":[6],"to":[7,127,155],"match":[8],"another":[9],"by":[10],"minimizing":[11],"their":[12],"Kullback-Leibler":[13],"(KL)":[14],"divergence,":[15],"is":[16],"widely":[17],"used":[18],"in":[19,135],"knowledge":[20],"model":[22],"compression,":[23],"continual":[24],"learning,":[25],"and":[26,45,114,129,138],"sparse-attention":[27],"LLM":[28],"training.":[29],"However,":[30],"existing":[31],"approaches":[32],"materialize":[33],"both":[34],"distributions":[36],"before":[37],"computing":[38],"the":[39,58,79,98,136,146],"KL":[40,65,82],"reduction,":[41,83],"incurring":[42],"$O(N_QN_K)$":[43,154],"memory":[44],"IO":[46],"costs":[47],"that":[48,67,90],"become":[49],"prohibitive":[50],"at":[51],"long":[52],"context":[53],"lengths.":[54],"We":[55,111],"present":[56],"StreamKL,":[57],"first":[59],"fused":[60],"GPU":[61,117],"primitive":[62],"for":[63,78],"divergence":[66],"eliminates":[68],"this":[69],"quadratic":[70,109],"materialization.":[71],"StreamKL":[72,101,124,144],"derives":[73],"a":[74,85,161],"novel":[75],"online":[76],"formulation":[77],"coupled":[80],"two-distribution":[81],"enabling":[84,157],"single":[86,162],"one-pass":[87],"forward":[88,137],"kernel":[89],"streams":[91],"query-key":[92],"tiles":[93],"through":[94],"on-chip":[95],"SRAM.":[96],"For":[97],"backward":[99,139],"pass,":[100],"recomputes":[102],"probabilities":[104],"tile-by-tile,":[105],"avoiding":[106],"storage":[107],"of":[108,150],"intermediates.":[110],"further":[112],"design":[113],"implement":[115],"efficient":[116],"kernels":[118],"with":[119],"dedicated":[120],"optimizations.":[121],"Experiments":[122],"show":[123],"delivers":[125],"up":[126],"$43\\times$":[128],"$14\\times$":[130],"speedups":[131],"over":[132],"baseline":[133],"methods":[134],"passes,":[140],"respectively.":[141],"Most":[142],"importantly,":[143],"reduces":[145],"extra":[147],"HBM":[148],"footprint":[149],"distillation":[152,159],"from":[153],"$O(1)$,":[156],"long-context":[158],"on":[160],"GPU.":[163]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-20T00:00:00"}