{"id":"https://openalex.org/W4417453645","doi":"https://doi.org/10.48550/arxiv.2512.13033","title":"Scaling Bidirectional Spans and Span Violations in Attention Mechanism","display_name":"Scaling Bidirectional Spans and Span Violations in Attention Mechanism","publication_year":2025,"publication_date":"2025-12-15","ids":{"openalex":"https://openalex.org/W4417453645","doi":"https://doi.org/10.48550/arxiv.2512.13033"},"language":null,"primary_location":{"id":"pmh:oai:arXiv.org:2512.13033","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2512.13033","pdf_url":"https://arxiv.org/pdf/2512.13033","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2512.13033","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100767186","display_name":"Jong-Wook Kim","orcid":"https://orcid.org/0000-0002-0379-6521"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Jongwook","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yun, Sangheon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yun, Sangheon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5001222858","display_name":"Suk-Jin Yoon","orcid":"https://orcid.org/0000-0002-1842-4325"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yoon, Sukjin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.258899986743927,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.258899986743927,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.20759999752044678,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.06790000200271606,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.7215999960899353},{"id":"https://openalex.org/keywords/projection","display_name":"Projection (relational algebra)","score":0.5382999777793884},{"id":"https://openalex.org/keywords/mechanism","display_name":"Mechanism (biology)","score":0.3878999948501587},{"id":"https://openalex.org/keywords/decomposition","display_name":"Decomposition","score":0.37229999899864197},{"id":"https://openalex.org/keywords/span","display_name":"Span (engineering)","score":0.3587000072002411},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.3578000068664551},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.3276999890804291},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.32440000772476196}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.7215999960899353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6151999831199646},{"id":"https://openalex.org/C57493831","wikidata":"https://www.wikidata.org/wiki/Q3134666","display_name":"Projection (relational algebra)","level":2,"score":0.5382999777793884},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.45500001311302185},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4092000126838684},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.3878999948501587},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.37229999899864197},{"id":"https://openalex.org/C2778753569","wikidata":"https://www.wikidata.org/wiki/Q1960395","display_name":"Span (engineering)","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.3578000068664551},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3276999890804291},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.32440000772476196},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.32249999046325684},{"id":"https://openalex.org/C25570617","wikidata":"https://www.wikidata.org/wiki/Q1006462","display_name":"Hilbert\u2013Huang transform","level":3,"score":0.3091000020503998},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.3037000000476837},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2906999886035919},{"id":"https://openalex.org/C175694140","wikidata":"https://www.wikidata.org/wiki/Q980329","display_name":"Orthographic projection","level":2,"score":0.28769999742507935},{"id":"https://openalex.org/C2778258933","wikidata":"https://www.wikidata.org/wiki/Q16918986","display_name":"Decomposition method (queueing theory)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C91682802","wikidata":"https://www.wikidata.org/wiki/Q620538","display_name":"Multidimensional scaling","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.25110000371932983}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2512.13033","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2512.13033","pdf_url":"https://arxiv.org/pdf/2512.13033","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2512.13033","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.13033","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2512.13033","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2512.13033","pdf_url":"https://arxiv.org/pdf/2512.13033","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4417453645.pdf","grobid_xml":"https://content.openalex.org/works/W4417453645.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"canonical":[1,47],"$O(N^2)$":[2],"Transformer":[3],"remains":[4],"the":[5,35,46,67,89,95,114],"empirical":[6],"performance":[7],"frontier":[8],"in":[9,110],"sequence":[10],"modeling,":[11],"and":[12,41,59,99,118,126],"its":[13],"training":[14,128],"can":[15],"be":[16],"further":[17],"optimized":[18],"by":[19],"addressing":[20],"geometric":[21],"inefficiency.":[22],"We":[23,73],"propose":[24],"an":[25,30],"optimization":[26],"framework":[27],"that":[28,75],"leverages":[29],"asymmetric":[31],"projection":[32,60],"to":[33],"decompose":[34],"backward-pass":[36],"gradients":[37],"into":[38],"parallel":[39,86],"spans":[40],"orthogonal":[42],"violations,":[43],"while":[44],"keeping":[45],"forward-pass":[48],"$QKV$":[49],"structure":[50],"intact.":[51],"Through":[52],"consistent":[53],"experimental":[54],"validation":[55,111],"across":[56],"various":[57],"decomposition":[58],"setups,":[61],"we":[62],"provide":[63],"strong":[64],"theoretical":[65],"evidence:":[66],"standard":[68],"attention":[69],"gradient":[70],"is":[71],"suboptimal.":[72],"demonstrated":[74],"selectively":[76],"scaling":[77],"these":[78],"components,":[79],"focusing":[80],"primarily":[81],"on":[82,123],"$0^{th}$":[83],"order":[84],"bidirectional":[85],"spans,":[87],"yields":[88],"most":[90],"effective":[91],"learning":[92],"signal.":[93],"On":[94],"limited":[96],"WikiText-2":[97],"dataset,":[98],"using":[100],"a":[101,107],"crude":[102],"configuration,":[103],"this":[104],"method":[105],"achieved":[106],"$0.56\\%$":[108],"reduction":[109],"loss,":[112],"confirming":[113],"framework's":[115],"fundamental":[116],"validity":[117],"suggesting":[119],"significant":[120],"potential":[121],"gains":[122],"larger":[124],"datasets":[125],"deeper":[127],"regimes":[129]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-12-17T00:00:00"}