{"id":"https://openalex.org/W7220713610","doi":"https://doi.org/10.48550/arxiv.2610.04424","title":"On the Trade-off Between Information Loss and Generalization in Sparse Attention","display_name":"On the Trade-off Between Information Loss and Generalization in Sparse Attention","publication_year":2026,"publication_date":"2026-10-03","ids":{"openalex":"https://openalex.org/W7220713610","doi":"https://doi.org/10.48550/arxiv.2610.04424"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.04424","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.04424","pdf_url":"https://arxiv.org/pdf/2610.04424","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.04424","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5152832091","display_name":"Zhongqi Fan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I12615008","display_name":"Beijing Normal-Hong Kong Baptist University","ror":"https://ror.org/04snvc712","country_code":"CN","type":"education","lineage":["https://openalex.org/I12615008"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fan, Zhongqi","raw_affiliation_strings":["Beijing Normal-Hong Kong Baptist University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Normal-Hong Kong Baptist University","institution_ids":["https://openalex.org/I12615008"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5157672466","display_name":"Zheng Tan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tan, Zheng","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.48100000619888306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.48100000619888306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.2410999983549118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.10849999636411667,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-attention","display_name":"sparse attention","score":0.9929999709129333},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.5680000185966492},{"id":"https://openalex.org/keywords/information-loss","display_name":"information loss","score":0.7379999756813049},{"id":"https://openalex.org/keywords/generalization","display_name":"generalization","score":0.7749999761581421},{"id":"https://openalex.org/keywords/jensen-shannon-divergence","display_name":"Jensen-Shannon divergence","score":0.9900000095367432},{"id":"https://openalex.org/keywords/approximation-error","display_name":"approximation error","score":0.7059999704360962},{"id":"https://openalex.org/keywords/rademacher-complexity","display_name":"Rademacher complexity","score":0.902999997138977},{"id":"https://openalex.org/keywords/mutual-information","display_name":"mutual information","score":0.7039999961853027},{"id":"https://openalex.org/keywords/generalization-bounds","display_name":"generalization bounds","score":0.7889999747276306},{"id":"https://openalex.org/keywords/quadratic-complexity","display_name":"quadratic complexity","score":0.5989999771118164}],"concepts":[{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.7786999940872192},{"id":"https://openalex.org/C60008888","wikidata":"https://www.wikidata.org/wiki/Q6031013","display_name":"Information bottleneck method","level":3,"score":0.7261999845504761},{"id":"https://openalex.org/C141513077","wikidata":"https://www.wikidata.org/wiki/Q378542","display_name":"Independent and identically distributed random variables","level":3,"score":0.5945000052452087},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.5077999830245972},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4578000009059906},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4465999901294708},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.44600000977516174},{"id":"https://openalex.org/C2776459999","wikidata":"https://www.wikidata.org/wiki/Q2119376","display_name":"Fidelity","level":2,"score":0.4359999895095825},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.4180999994277954},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.4018000066280365},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.40049999952316284},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.38510000705718994},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.3686000108718872},{"id":"https://openalex.org/C52622258","wikidata":"https://www.wikidata.org/wiki/Q131222","display_name":"Information theory","level":2,"score":0.3582000136375427},{"id":"https://openalex.org/C106195933","wikidata":"https://www.wikidata.org/wiki/Q7847935","display_name":"Truncation (statistics)","level":2,"score":0.35600000619888306},{"id":"https://openalex.org/C122123141","wikidata":"https://www.wikidata.org/wiki/Q176623","display_name":"Random variable","level":2,"score":0.28220000863075256},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C151201525","wikidata":"https://www.wikidata.org/wiki/Q177239","display_name":"Limit (mathematics)","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.27480000257492065},{"id":"https://openalex.org/C148764684","wikidata":"https://www.wikidata.org/wiki/Q621751","display_name":"Approximation algorithm","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C110121322","wikidata":"https://www.wikidata.org/wiki/Q865811","display_name":"Distribution (mathematics)","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C167981619","wikidata":"https://www.wikidata.org/wiki/Q1685498","display_name":"Cross entropy","level":3,"score":0.2549999952316284}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.04424","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.04424","pdf_url":"https://arxiv.org/pdf/2610.04424","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.04424","doi":"https://doi.org/10.48550/arxiv.2610.04424","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.04424","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.04424","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.04424","pdf_url":"https://arxiv.org/pdf/2610.04424","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7220713610.pdf","grobid_xml":"https://content.openalex.org/works/W7220713610.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W2146950091","https://openalex.org/W2626778328","https://openalex.org/W2887997457","https://openalex.org/W2940744433","https://openalex.org/W2994673210","https://openalex.org/W2997517014","https://openalex.org/W3015468748","https://openalex.org/W3033529678","https://openalex.org/W3037932933","https://openalex.org/W3045733172","https://openalex.org/W3094502228","https://openalex.org/W3172946450","https://openalex.org/W4250954493","https://openalex.org/W4301425558","https://openalex.org/W4378232390","https://openalex.org/W4387891214","https://openalex.org/W4409670972","https://openalex.org/W4412944812","https://openalex.org/W4415308159","https://openalex.org/W4416047632","https://openalex.org/W4416113689","https://openalex.org/W4416536837","https://openalex.org/W6929270298","https://openalex.org/W7101917293","https://openalex.org/W7123877496","https://openalex.org/W7128604689","https://openalex.org/W7131450452","https://openalex.org/W7166822821"],"related_works":["https://openalex.org/W4415109377","https://openalex.org/W7133183975","https://openalex.org/W7075979799","https://openalex.org/W3204470331","https://openalex.org/W4415308159","https://openalex.org/W4387031204","https://openalex.org/W7160854219","https://openalex.org/W4395066850","https://openalex.org/W4417144098","https://openalex.org/W4304699881"],"abstract_inverted_index":{"To":[0,65],"mitigate":[1],"the":[2,7,18,25,45,59,63,76,82,93,97,106,112,130,134,139,147,199,205,222,225,238,248],"quadratic":[3],"complexity":[4,223],"bottleneck":[5],"of":[6,22,28,48,62,75,81,85,105,198,224,247],"Transformer,":[8],"sparse":[9,23,29,86,140,200,256],"attention":[10,30,49,87,113,136,141],"has":[11],"emerged":[12],"as":[13],"a":[14,72,169,187,244],"pivotal":[15],"technology.":[16],"Despite":[17],"extensive":[19],"empirical":[20],"success":[21],"Transformers,":[24],"theoretical":[26,245],"understanding":[27],"remains":[31],"fragmented.":[32],"In":[33],"particular,":[34],"two":[35],"fundamental":[36],"questions":[37],"remain":[38],"unclear:":[39],"(1)":[40],"How":[41,52],"does":[42,53],"sparsification":[43],"affect":[44],"information":[46,55,251],"fidelity":[47,252],"mechanisms?":[50],"(2)":[51],"this":[54,67,69],"loss":[56],"interact":[57],"with":[58,126,192],"generalization":[60,83,170,189,207,254],"behavior":[61],"model?":[64],"bridge":[66],"gap,":[68],"paper":[70],"proposes":[71],"systematic":[73],"analysis":[74,104,197,217],"Jensen-Shannon":[77],"(JS)":[78],"divergence":[79,132],"and":[80,120,138,195,253],"gap":[84],"mechanisms.":[88],"Specifically,":[89],"we":[90,167,203],"first":[91],"characterize":[92],"approximation":[94,231],"error":[95,232],"via":[96],"JS":[98,131,239],"divergence.":[99,240],"Through":[100],"an":[101,193],"order-statistics-based":[102],"concentration":[103],"truncation":[107],"mass":[108],"alpha":[109],"---":[110,129],"where":[111],"scores":[114],"are":[115],"assumed":[116],"to":[117,145],"be":[118,235],"independent":[119],"identically":[121],"distributed":[122],"sub-Gaussian":[123],"random":[124],"variables":[125],"parameter":[127],"sigma":[128],"between":[133,250],"full":[135],"distribution":[137,142],"is":[143],"shown":[144],"admit":[146],"closed":[148],"form":[149],"log":[150],"2":[151],"+":[152,182,212,214],"((1":[153],"-":[154,157,159,161,164],"alpha)/2)":[155,162],"log(1":[156,213],"alpha)":[158],"((2":[160],"log(2":[163],"alpha).":[165],"Subsequently,":[166],"derive":[168],"bound":[171,190,208],"through":[172],"Rademacher":[173],"complexity,":[174],"quantified":[175,236],"by":[176,237],"O(gamma":[177],"*":[178,180,210],"sqrt(M/n)":[179],"(sqrt(log(3eL/M))":[181],"sqrt(pi)/2)).":[183],"Furthermore,":[184],"building":[185],"on":[186],"mutual-information-based":[188],"together":[191],"entropy":[194],"covering-number":[196],"hypothesis":[201,227],"class,":[202],"obtain":[204],"sparsity-dependent":[206],"O(sqrt((M/(2n))":[209],"(log(eL/M)":[211],"2/epsilon)))).":[215],"Our":[216],"shows":[218],"that":[219,233],"sparsity":[220],"reduces":[221],"considered":[226],"class":[228],"while":[229],"introducing":[230],"can":[234],"These":[241],"findings":[242],"provide":[243],"characterization":[246],"trade-off":[249],"in":[255],"Transformer":[257],"architectures.":[258]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-10-07T00:00:00"}