{"id":"https://openalex.org/W7204883366","doi":"https://doi.org/10.48550/arxiv.2608.30417","title":"Inherent Trade-offs when Enforcing Correct Data Symmetries in Multi-head Self-attention","display_name":"Inherent Trade-offs when Enforcing Correct Data Symmetries in Multi-head Self-attention","publication_year":2026,"publication_date":"2026-08-31","ids":{"openalex":"https://openalex.org/W7204883366","doi":"https://doi.org/10.48550/arxiv.2608.30417"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2608.30417","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30417","pdf_url":"https://arxiv.org/pdf/2608.30417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2608.30417","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139843062","display_name":"T\u012bkun \u00d4ng","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u00d4ng, T\u012bkun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5084356187","display_name":"Georg B\u00f6kman","orcid":"https://orcid.org/0000-0001-7522-2255","observed_orcids":["https://orcid.org/0000-0001-7522-2255"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"B\u00f6kman, Georg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.14169999957084656,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.14169999957084656,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.09830000251531601,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0940999984741211,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-head-self-attention","display_name":"multi-head self-attention","score":0.9819999933242798},{"id":"https://openalex.org/keywords/equivariance","display_name":"equivariance","score":0.8550000190734863},{"id":"https://openalex.org/keywords/symmetry-groups","display_name":"symmetry groups","score":0.7200000286102295},{"id":"https://openalex.org/keywords/parameter-space","display_name":"parameter space","score":0.6259999871253967},{"id":"https://openalex.org/keywords/d4-symmetry","display_name":"D4 symmetry","score":0.7910000085830688},{"id":"https://openalex.org/keywords/regular-representation","display_name":"regular representation","score":0.640999972820282}],"concepts":[{"id":"https://openalex.org/C171036898","wikidata":"https://www.wikidata.org/wiki/Q256355","display_name":"Equivariant map","level":2,"score":0.957099974155426},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.631600022315979},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.4699999988079071},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4388999938964844},{"id":"https://openalex.org/C2781311116","wikidata":"https://www.wikidata.org/wiki/Q83306","display_name":"Group (periodic table)","level":2,"score":0.43320000171661377},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.4097999930381775},{"id":"https://openalex.org/C2779886137","wikidata":"https://www.wikidata.org/wiki/Q21030012","display_name":"Symmetry (geometry)","level":2,"score":0.35249999165534973},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.3395000100135803},{"id":"https://openalex.org/C136119220","wikidata":"https://www.wikidata.org/wiki/Q1000660","display_name":"Algebra over a field","level":2,"score":0.3352999985218048},{"id":"https://openalex.org/C44306375","wikidata":"https://www.wikidata.org/wiki/Q902019","display_name":"Symmetry group","level":2,"score":0.3305000066757202},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.3269999921321869},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3215000033378601},{"id":"https://openalex.org/C2780841128","wikidata":"https://www.wikidata.org/wiki/Q5073781","display_name":"Characterization (materials science)","level":2,"score":0.31839999556541443},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.30160000920295715},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C2777742833","wikidata":"https://www.wikidata.org/wiki/Q1964083","display_name":"Reciprocal","level":2,"score":0.2687999904155731},{"id":"https://openalex.org/C96469262","wikidata":"https://www.wikidata.org/wiki/Q1324364","display_name":"Homogeneous space","level":2,"score":0.26429998874664307},{"id":"https://openalex.org/C84597430","wikidata":"https://www.wikidata.org/wiki/Q106227","display_name":"Locus (genetics)","level":3,"score":0.257999986410141},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.2531999945640564}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2608.30417","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30417","pdf_url":"https://arxiv.org/pdf/2608.30417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2608.30417","doi":"https://doi.org/10.48550/arxiv.2608.30417","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2608.30417","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2608.30417","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2608.30417","pdf_url":"https://arxiv.org/pdf/2608.30417","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7204883366.pdf","grobid_xml":"https://content.openalex.org/works/W7204883366.grobid-xml"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W2092692582","https://openalex.org/W2279221249","https://openalex.org/W2785397276","https://openalex.org/W2796963594","https://openalex.org/W2799020113","https://openalex.org/W2825179178","https://openalex.org/W2900364424","https://openalex.org/W3008803199","https://openalex.org/W3036737467","https://openalex.org/W3089607359","https://openalex.org/W3169316583","https://openalex.org/W3171974924","https://openalex.org/W4226434736","https://openalex.org/W4231799185","https://openalex.org/W4281253669","https://openalex.org/W4380558414","https://openalex.org/W4403524676","https://openalex.org/W4404358865","https://openalex.org/W4407093434","https://openalex.org/W4407310642","https://openalex.org/W4410361949","https://openalex.org/W4414969228","https://openalex.org/W4415885765","https://openalex.org/W6979223214","https://openalex.org/W7066499495","https://openalex.org/W7126196520","https://openalex.org/W7128524502","https://openalex.org/W7156752337","https://openalex.org/W7166573401"],"related_works":["https://openalex.org/W7220995153","https://openalex.org/W3089607359","https://openalex.org/W4366000772","https://openalex.org/W4385570922","https://openalex.org/W7162000890","https://openalex.org/W7203890268","https://openalex.org/W7116984659","https://openalex.org/W7206178128","https://openalex.org/W7221529798","https://openalex.org/W7107866822"],"abstract_inverted_index":{"We":[0,127],"give":[1],"a":[2,17,44,81,89],"complete":[3],"characterization":[4],"of":[5,71,77,83,107],"equivariant":[6,15,72],"multi-head":[7],"self-attention":[8],"(MHSA):":[9],"if":[10],"an":[11,35],"MHSA":[12,51,61,79],"layer":[13,148],"is":[14,149],"to":[16,39,65,151],"symmetry":[18],"group":[19,41],"$G$,":[20],"then":[21],"$G$":[22],"can":[23,142],"only":[24],"act":[25],"by":[26,57],"permuting":[27],"head-clusters,":[28],"with":[29,130],"QK":[30],"and":[31,93,135],"OV":[32],"matrices":[33],"satisfying":[34],"equivariance":[36,56,75],"constraint":[37],"tied":[38],"the":[40,69,74,108,112,138,146],"action.":[42],"As":[43],"consequence,":[45],"we":[46,116],"prove":[47],"that":[48,53,118,137],"any":[49,94],"fixed":[50],"architecture":[52,96],"achieves":[54],"exact":[55],"polynomially":[58],"parameterizing":[59],"unconstrained":[60,78],"parameters":[62],"inevitably":[63],"leads":[64],"expressivity":[66],"loss":[67],"within":[68],"class":[70],"maps:":[73],"locus":[76],"forms":[80],"union":[82],"extremely":[84],"many":[85],"Zariski-irreducible":[86],"components":[87,122],"in":[88],"reduced":[90],"parameter":[91],"space,":[92,115],"single":[95],"covers":[97],"at":[98],"most":[99],"one.":[100],"For":[101],"$G=D_4$":[102],"acting":[103],"on":[104,132],"$C$":[105],"copies":[106],"regular":[109],"representation":[110],"as":[111],"token":[113],"feature":[114],"show":[117,136],"there":[119],"are":[120],"$\u03a9(C^{64})$":[121],"for":[123],"eight":[124],"attention":[125,147],"heads.":[126],"illustrate":[128],"this":[129],"experiments":[131],"artificial":[133],"tasks":[134],"predicted":[139],"head-cluster":[140],"structures":[141],"arise":[143],"naturally":[144],"when":[145],"pressured":[150],"become":[152],"equivariant.":[153]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-09-02T00:00:00"}