{"id":"https://openalex.org/W7224831233","doi":"https://doi.org/10.5281/zenodo.23790300","title":"Beyond Proportional Allocation: Optimal Circuit-Coupled Rank Assignment in Multi-Head Attention via Marginal Distortion Reduction and Gauge Concavity","display_name":"Beyond Proportional Allocation: Optimal Circuit-Coupled Rank Assignment in Multi-Head Attention via Marginal Distortion Reduction and Gauge Concavity","publication_year":2026,"publication_date":"2026-10-30","ids":{"openalex":"https://openalex.org/W7224831233","doi":"https://doi.org/10.5281/zenodo.23790300"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23790300","doi":"https://doi.org/10.5281/zenodo.23790300","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23790300","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23790300","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148599496","display_name":"Hideki Ishiyama","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ishiyama, Hideki","raw_affiliation_strings":["identical reseacher"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"identical reseacher","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5148599496"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9185000061988831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9185000061988831,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.024700000882148743,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.023000000044703484,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-head-attention","display_name":"multi-head attention","score":0.9810000061988831},{"id":"https://openalex.org/keywords/low-rank-compression","display_name":"low-rank compression","score":0.9279999732971191},{"id":"https://openalex.org/keywords/water-filling","display_name":"water-filling","score":0.7369999885559082},{"id":"https://openalex.org/keywords/degrees-of-freedom","display_name":"degrees of freedom","score":0.7680000066757202},{"id":"https://openalex.org/keywords/lora","display_name":"LoRa","score":0.8880000114440918},{"id":"https://openalex.org/keywords/gpt-2","display_name":"GPT-2","score":0.8090000152587891}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.8206999897956848},{"id":"https://openalex.org/C199422724","wikidata":"https://www.wikidata.org/wiki/Q41118","display_name":"Quotient","level":2,"score":0.47360000014305115},{"id":"https://openalex.org/C70518039","wikidata":"https://www.wikidata.org/wiki/Q16000077","display_name":"Dimensionality reduction","level":2,"score":0.47350001335144043},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.46860000491142273},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.4553999900817871},{"id":"https://openalex.org/C22324862","wikidata":"https://www.wikidata.org/wiki/Q652707","display_name":"Lipschitz continuity","level":2,"score":0.44200000166893005},{"id":"https://openalex.org/C70710897","wikidata":"https://www.wikidata.org/wiki/Q680081","display_name":"Separable space","level":2,"score":0.4359000027179718},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.426800012588501},{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.4056999981403351},{"id":"https://openalex.org/C77246614","wikidata":"https://www.wikidata.org/wiki/Q1409400","display_name":"Gramian matrix","level":3,"score":0.39469999074935913},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.390500009059906},{"id":"https://openalex.org/C73684929","wikidata":"https://www.wikidata.org/wiki/Q598870","display_name":"Lagrange multiplier","level":2,"score":0.3871999979019165},{"id":"https://openalex.org/C181789720","wikidata":"https://www.wikidata.org/wiki/Q4812191","display_name":"Asymptotically optimal algorithm","level":2,"score":0.3871000111103058},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3813999891281128},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.36059999465942383},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3522000014781952},{"id":"https://openalex.org/C41341539","wikidata":"https://www.wikidata.org/wiki/Q7540242","display_name":"Sliced inverse regression","level":3,"score":0.3463999927043915},{"id":"https://openalex.org/C158968445","wikidata":"https://www.wikidata.org/wiki/Q7631150","display_name":"Subgradient method","level":2,"score":0.3411000072956085},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.335099995136261},{"id":"https://openalex.org/C126780896","wikidata":"https://www.wikidata.org/wiki/Q899871","display_name":"Distortion (music)","level":4,"score":0.329800009727478},{"id":"https://openalex.org/C27931671","wikidata":"https://www.wikidata.org/wiki/Q7634497","display_name":"Sufficient dimension reduction","level":3,"score":0.31940001249313354},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.3138999938964844},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.31139999628067017},{"id":"https://openalex.org/C22789450","wikidata":"https://www.wikidata.org/wiki/Q420904","display_name":"Singular value decomposition","level":2,"score":0.2791000008583069},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.27140000462532043},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.2540999948978424}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23790300","doi":"https://doi.org/10.5281/zenodo.23790300","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23790300","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23790301","doi":"https://doi.org/10.5281/zenodo.23790301","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23790301","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23790300","doi":"https://doi.org/10.5281/zenodo.23790300","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23790300","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7165626211","https://openalex.org/W4414695225","https://openalex.org/W7220474976","https://openalex.org/W4416034622","https://openalex.org/W4416330253","https://openalex.org/W7203724586","https://openalex.org/W7138903718","https://openalex.org/W4415344136","https://openalex.org/W7214246234","https://openalex.org/W7162000890"],"abstract_inverted_index":{"Existing":[0],"low-rank":[1],"compression":[2],"methods":[3],"for":[4,123],"Multi-Head":[5],"Attention":[6],"(MHA),":[7],"such":[8],"as":[9],"LoRA":[10],"and":[11,46,219],"SVD":[12,71],"pruning,":[13],"evaluate":[14],"weight":[15],"matrices":[16],"in":[17],"isolation.":[18],"We":[19],"demonstrate":[20],"that":[21,96,162,204],"this":[22,53],"isolated":[23,43,91],"evaluation":[24,185],"creates":[25],"a":[26,31,57,111,131,190],"single-matrix":[27],"blind":[28],"spot,":[29],"yielding":[30],"negative":[32],"correlation":[33],"(\\(r":[34,78],"=":[35,79,155],"-0.1582\\)":[36],"on":[37,186,227],"OpenAI":[38],"GPT-2":[39,187],"Layer":[40,209],"#0)":[41],"between":[42],"truncation":[44],"energy":[45],"actual":[47],"MHA":[48,60],"output":[49,83,211],"distortion.":[50],"To":[51],"address":[52],"limitation,":[54],"we":[55,94,116,143],"establish":[56],"provable":[58],"Composite":[59],"Distortion":[61],"Bound":[62],"combining":[63],"row-wise":[64],"Softmax":[65],"Lipschitz":[66],"contraction":[67],"with":[68,81],"input":[69],"covariance-weighted":[70],"trace":[72],"bounds.":[73],"This":[74],"bound":[75],"strongly":[76],"correlates":[77],"+0.9680\\))":[80],"forward":[82],"distortion,":[84],"accurately":[85],"identifying":[86],"critical":[87],"heads":[88],"misclassified":[89],"by":[90,171,216],"metrics.":[92],"Furthermore,":[93],"prove":[95,117,144],"conventional":[97],"proportional":[98],"allocation":[99,164],"(\\(s_m":[100],"\\propto":[101],"C_m\\))":[102],"is":[103],"sub-optimal":[104],"due":[105],"to":[106,197,223,232],"heterogeneous":[107],"spectral":[108],"decay.":[109],"Formulating":[110],"separable":[112],"surrogate":[113],"distortion":[114],"objective,":[115],"the":[118,145],"Discrete":[119],"Marginal":[120],"Allocation":[121],"Theorem":[122],"Separable":[124],"Surrogate":[125],"Objectives:":[126],"under":[127,189],"non-increasing":[128],"marginal":[129,206],"reductions,":[130],"greedy":[132],"Lagrangian":[133],"water-filling":[134],"algorithm":[135],"achieves":[136],"an":[137],"exact":[138],"integer":[139],"global":[140],"optimum.":[141],"Additionally,":[142],"Gauge-Concavity":[146],"Identity:":[147],"strict":[148],"concavity":[149],"of":[150,168],"quotient":[151,192],"manifold":[152,193],"dimension":[153,194],"\\(p_{\\text{eff}}(s)":[154],"s(d_{\\text{out}}":[156],"+":[157],"d_{\\text{in}}":[158],"-":[159,174],"s)\\)":[160],"implies":[161],"non-uniform":[163],"reduces":[165,208],"geometric":[166],"degrees":[167],"freedom":[169],"(DoF)":[170],"\\(\\sum":[172],"(s_m":[173],"\\bar{s})^2\\),":[175],"enabling":[176],"DoF":[177],"reinvestment":[178],"regimes":[179],"(Modes":[180],"A,":[181],"B1,":[182],"B2).":[183],"Empirical":[184],"(124M)":[188],"73.44%":[191],"reduction":[195],"(corresponding":[196],"72.92%":[198],"physical":[199],"storage":[200],"parameter":[201],"reduction)":[202],"shows":[203],"discrete":[205],"optimization":[207],"#0":[210],"Mean":[212],"Squared":[213],"Error":[214],"(MSE)":[215],"96.16%":[217],"in-sample":[218],"81.03%":[220],"(Mode":[221,225],"A/B2)":[222],"83.65%":[224],"B1)":[226],"held-out":[228],"test":[229],"sequences":[230],"compared":[231],"uniform":[233],"allocation.":[234]},"counts_by_year":[],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2026-10-31T00:00:00"}