{"id":"https://openalex.org/W7167612071","doi":"https://doi.org/10.48550/arxiv.2607.02584","title":"RotateAttention: RoPE-Aware Rotation and Range Rectification for INT4 Quantized Attention in Video Generation","display_name":"RotateAttention: RoPE-Aware Rotation and Range Rectification for INT4 Quantized Attention in Video Generation","publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7167612071","doi":"https://doi.org/10.48550/arxiv.2607.02584"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.02584","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02584","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.02584","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5121806168","display_name":"Yaofu Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yaofu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140185690","display_name":"Wanli Lan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lan, Wanli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140199149","display_name":"Jinxi Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jinxi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5140207770","display_name":"Binhang Yuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuan, Binhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5140197305","display_name":"Harry Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Harry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.3127000033855438,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.3127000033855438,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.05270000174641609,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.05139999836683273,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/quantization","display_name":"Quantization (signal processing)","score":0.5773000121116638},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.5569999814033508},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.5515999794006348},{"id":"https://openalex.org/keywords/computational-complexity-theory","display_name":"Computational complexity theory","score":0.5098999738693237},{"id":"https://openalex.org/keywords/speedup","display_name":"Speedup","score":0.5006999969482422},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.4560999870300293},{"id":"https://openalex.org/keywords/rotation","display_name":"Rotation (mathematics)","score":0.4399999976158142},{"id":"https://openalex.org/keywords/rectification","display_name":"Rectification","score":0.4277999997138977},{"id":"https://openalex.org/keywords/quadratic-equation","display_name":"Quadratic equation","score":0.4189000129699707}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6614999771118164},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.590499997138977},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.5773000121116638},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.5569999814033508},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.5515999794006348},{"id":"https://openalex.org/C179799912","wikidata":"https://www.wikidata.org/wiki/Q205084","display_name":"Computational complexity theory","level":2,"score":0.5098999738693237},{"id":"https://openalex.org/C68339613","wikidata":"https://www.wikidata.org/wiki/Q1549489","display_name":"Speedup","level":2,"score":0.5006999969482422},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.4560999870300293},{"id":"https://openalex.org/C74050887","wikidata":"https://www.wikidata.org/wiki/Q848368","display_name":"Rotation (mathematics)","level":2,"score":0.4399999976158142},{"id":"https://openalex.org/C50942859","wikidata":"https://www.wikidata.org/wiki/Q4967193","display_name":"Rectification","level":3,"score":0.4277999997138977},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.4189000129699707},{"id":"https://openalex.org/C83633838","wikidata":"https://www.wikidata.org/wiki/Q1256564","display_name":"Rotation matrix","level":2,"score":0.398499995470047},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3774999976158142},{"id":"https://openalex.org/C17349429","wikidata":"https://www.wikidata.org/wiki/Q1049914","display_name":"Matrix multiplication","level":3,"score":0.35530000925064087},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.33959999680519104},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C106487976","wikidata":"https://www.wikidata.org/wiki/Q685816","display_name":"Matrix (chemical analysis)","level":2,"score":0.32829999923706055},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.31369999051094055},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3021000027656555},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.3005000054836273},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.27889999747276306},{"id":"https://openalex.org/C103910844","wikidata":"https://www.wikidata.org/wiki/Q2631256","display_name":"Video quality","level":3,"score":0.26339998841285706},{"id":"https://openalex.org/C739882","wikidata":"https://www.wikidata.org/wiki/Q3560506","display_name":"Anomaly detection","level":2,"score":0.258899986743927},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.25839999318122864},{"id":"https://openalex.org/C56372850","wikidata":"https://www.wikidata.org/wiki/Q1050404","display_name":"Sparse matrix","level":3,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.02584","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02584","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.02584","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.02584","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0,99],"$\\textbf{DiT-based":[1,137],"video":[2,138,213],"generation":[3,139,214],"models":[4,140],"equipped":[5],"with":[6,26,76,141,204],"3D":[7,142],"Rotary":[8],"Position":[9],"Embeddings":[10],"(3D":[11],"RoPE)}$,":[12],"the":[13,80,95,105,116,200],"attention":[14,82,150],"mechanism":[15],"remains":[16],"a":[17,35,58],"primary":[18],"computational":[19,206],"bottleneck":[20],"due":[21],"to":[22,28,74,178,197,218,224],"its":[23],"quadratic":[24],"complexity":[25],"respect":[27],"sequence":[29],"length.":[30],"While":[31],"quantized":[32],"$\\textbf{FlashAttention}$":[33],"offers":[34],"promising":[36],"path":[37],"toward":[38],"hardware":[39],"acceleration,":[40],"existing":[41],"low-bit":[42],"quantization":[43,91],"methods":[44],"overlook":[45],"two":[46,157],"critical":[47],"challenges":[48],"in":[49,65,182],"this":[50,100,124],"setting:":[51],"$\\textbf{1)}$":[52],"applying":[53],"online":[54],"rotation":[55,167],"matrices":[56,168,177],"--":[57,71],"widely":[59],"used":[60],"technique":[61],"for":[62,136,148],"mitigating":[63],"outliers":[64,181],"Queries":[66],"($Q$)":[67],"and":[68,78,110,152,184,186,195,228],"Keys":[69],"($K$)":[70],"is":[72],"difficult":[73],"reconcile":[75],"$\\textbf{RoPE}$;":[77],"$\\textbf{2)}$":[79],"non-negative":[81],"matrix":[83],"$P":[84],"=":[85],"\\exp(QK":[86],"-":[87],"\\max(QK))$":[88],"makes":[89],"symmetric":[90],"waste":[92],"half":[93],"of":[94,108,119],"4-bit":[96],"dynamic":[97],"range.":[98],"work,":[101],"we":[102,126],"observe":[103],"that":[104,169,210],"outlier":[106],"distributions":[107],"$Q$":[109,183],"$K$":[111],"are":[112],"strongly":[113],"affected":[114],"by":[115],"dimensional":[117],"partitioning":[118],"$\\textbf{3D":[120],"RoPE}$.":[121],"Based":[122],"on":[123],"finding,":[125],"propose":[127],"$\\textbf{RotateAttention}$,":[128],"an":[129],"efficient":[130],"$\\textbf{mixed-precision":[131],"INT4":[132],"FlashAttention}$":[133],"framework":[134],"tailored":[135],"RoPE}$,":[143],"using":[144],"selective":[145],"$\\textbf{FP16":[146],"fallback}$":[147],"accuracy-sensitive":[149],"blocks":[151],"denoising":[153],"steps.":[154],"RotateAttention":[155],"introduces":[156],"core":[158],"techniques:":[159],"$\\textbf{1)":[160],"RoPE-aware":[161],"Rotation}$,":[162],"which":[163,191],"employs":[164],"either":[165],"mergeable":[166],"can":[170],"be":[171],"fused":[172],"into":[173],"RoPE":[174],"or":[175],"negligible-overhead":[176],"mitigate":[179],"RoPE-induced":[180],"$K$;":[185],"$\\textbf{2)":[187],"Range-optimized":[188],"$P$":[189],"Quantization}$,":[190],"uses":[192],"fixed":[193],"scales":[194],"zero-points":[196],"fully":[198],"exploit":[199],"$\\textbf{INT4":[201],"numerical":[202],"range}$":[203],"minimal":[205],"overhead.":[207],"Experiments":[208],"show":[209],"$\\textbf{RotateAttention}$":[211],"preserves":[212],"quality":[215],"nearly":[216],"identical":[217],"full-precision":[219],"baselines":[220],"while":[221],"achieving":[222],"up":[223],"1.68$\\times$":[225],"end-to-end":[226],"speedup":[227],"2.2$\\times$":[229],"kernel-level":[230],"acceleration.":[231]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-08T00:00:00"}