{"id":"https://openalex.org/W7213583145","doi":"https://doi.org/10.48550/arxiv.2609.20269","title":"Placement Is Free, Composition Is Not: The Latin Square as a Provably-Balanced Construction for Heterogeneous Sequence-Mixer Stacks","display_name":"Placement Is Free, Composition Is Not: The Latin Square as a Provably-Balanced Construction for Heterogeneous Sequence-Mixer Stacks","publication_year":2026,"publication_date":"2026-07-29","ids":{"openalex":"https://openalex.org/W7213583145","doi":"https://doi.org/10.48550/arxiv.2609.20269"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.20269","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20269","pdf_url":"https://arxiv.org/pdf/2609.20269","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.20269","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018517760","display_name":"Taebong Kim","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Taebong","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136266590","display_name":"Youngsik Hong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hong, Youngsik","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100629728","display_name":"Minsik Kim","orcid":"https://orcid.org/0000-0002-0502-5464","observed_orcids":["https://orcid.org/0000-0002-0502-5464"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Minsik","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133541420","display_name":"Sunyoung CHOI","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Sunyoung","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148103775","display_name":"Jaewon Jang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jang, Jaewon","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5148090564","display_name":"Minseo Kim","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Minseo","raw_affiliation_strings":["VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"VIDRAFT AI Research \u2022 QuantumOS , Seoul , Republic of Korea","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.82669997215271,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.82669997215271,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.054999999701976776,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.020099999383091927,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mixture-of-experts","display_name":"mixture of experts","score":0.7490000128746033},{"id":"https://openalex.org/keywords/latin-squares","display_name":"Latin squares","score":0.6420000195503235},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.33899998664855957},{"id":"https://openalex.org/keywords/state-space-models","display_name":"state-space model","score":0.7390000224113464}],"concepts":[{"id":"https://openalex.org/C66882249","wikidata":"https://www.wikidata.org/wiki/Q169336","display_name":"Homogeneous","level":2,"score":0.5242999792098999},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5153999924659729},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.5115000009536743},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.47200000286102295},{"id":"https://openalex.org/C9395851","wikidata":"https://www.wikidata.org/wiki/Q177929","display_name":"Stack (abstract data type)","level":2,"score":0.39590001106262207},{"id":"https://openalex.org/C158887663","wikidata":"https://www.wikidata.org/wiki/Q679367","display_name":"Latin square","level":4,"score":0.38830000162124634},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.34200000762939453},{"id":"https://openalex.org/C40231798","wikidata":"https://www.wikidata.org/wiki/Q1333743","display_name":"Composition (language)","level":2,"score":0.31869998574256897},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.310699999332428},{"id":"https://openalex.org/C99821215","wikidata":"https://www.wikidata.org/wiki/Q1136583","display_name":"Swap (finance)","level":2,"score":0.31049999594688416},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3084000051021576},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.30230000615119934},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.29739999771118164},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.2870999872684479},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.27489998936653137},{"id":"https://openalex.org/C135692309","wikidata":"https://www.wikidata.org/wiki/Q111124","display_name":"Square (algebra)","level":2,"score":0.26899999380111694},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.2639999985694885},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.25679999589920044},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.2513999938964844}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.20269","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20269","pdf_url":"https://arxiv.org/pdf/2609.20269","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.20269","doi":"https://doi.org/10.48550/arxiv.2609.20269","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.20269","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.20269","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.20269","pdf_url":"https://arxiv.org/pdf/2609.20269","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7213583145.pdf","grobid_xml":"https://content.openalex.org/works/W7213583145.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W3015468748","https://openalex.org/W3033529678","https://openalex.org/W4224308101","https://openalex.org/W4322718191","https://openalex.org/W4377865076","https://openalex.org/W4384648484","https://openalex.org/W4384648639","https://openalex.org/W4389326242","https://openalex.org/W4393399080","https://openalex.org/W4396815229","https://openalex.org/W4415795940","https://openalex.org/W7133225605"],"related_works":["https://openalex.org/W7223558060","https://openalex.org/W2988945824","https://openalex.org/W7223564422","https://openalex.org/W7170146559","https://openalex.org/W7213345024","https://openalex.org/W7164487019","https://openalex.org/W6941353567","https://openalex.org/W4400703021","https://openalex.org/W7166831317","https://openalex.org/W7163582471"],"abstract_inverted_index":{"Since":[0],"GPT,":[1],"most":[2],"Transformers":[3],"have":[4],"repeated":[5],"the":[6,80,155,168,209,217],"same":[7,156],"attention":[8],"mechanism":[9,38,71,219],"at":[10,202],"every":[11,76],"layer.":[12],"Yet":[13],"this":[14,93,200],"design":[15,81],"is":[16],"largely":[17],"a":[18,22,50,65,97,105,126,130,135,163,172,176,221,235],"convention":[19],"rather":[20,192],"than":[21,193],"tested":[23],"conclusion.":[24],"When":[25],"multiple":[26],"sequence":[27],"mixers":[28],"are":[29],"combined":[30],"in":[31,75],"one":[32,174],"stack,":[33],"improvements":[34],"may":[35],"arise":[36],"from":[37],"choice,":[39],"placement,":[40],"or":[41],"both,":[42],"making":[43],"causal":[44],"attribution":[45],"difficult.":[46],"We":[47,198,224,242],"introduce":[48],"Aether-7B-5Attn,":[49],"6.59B-parameter":[51],"mixture-of-experts":[52],"model":[53,244],"($\\approx$2.98B":[54],"active)":[55],"whose":[56],"49":[57,240],"layers":[58],"contain":[59],"seven":[60],"sequence-mixing":[61],"mechanisms":[62,102,157],"arranged":[63,103],"as":[64,104],"$7\\times7$":[66],"Latin":[67,107],"square.":[68],"Because":[69],"each":[70],"appears":[72],"exactly":[73],"once":[74],"row":[77],"and":[78,116,215,231,234,251],"column,":[79],"guarantees":[82],"balanced":[83,136],"exposure":[84],"across":[85,190],"depth":[86,160,191],"while":[87,166],"eliminating":[88],"placement":[89,148],"confounds.":[90],"To":[91],"evaluate":[92],"principle,":[94],"we":[95],"build":[96],"parameter-matched":[98],"proxy":[99],"with":[100,118,171],"four":[101],"$4\\times4$":[106],"square":[108],"over":[109],"sixteen":[110],"layers,":[111],"matched":[112],"to":[113,213],"700.9M":[114],"parameters":[115],"trained":[117],"eight":[119],"seeds":[120],"per":[121],"arm.":[122],"The":[123],"results":[124,180],"reveal":[125],"clear":[127],"dissociation.":[128],"Rearranging":[129],"distributed":[131,189],"heterogeneous":[132,169,187],"stack":[133,170],"into":[134,158],"periodic":[137],"cycle":[138],"changes":[139],"validation":[140],"loss":[141],"by":[142],"only":[143],"0.16\\%,":[144],"indicating":[145],"that":[146,182],"exact":[147],"has":[149],"little":[150],"effect.":[151],"In":[152],"contrast,":[153],"clustering":[154],"contiguous":[159],"bands":[161],"incurs":[162,175],"0.59\\%":[164],"penalty,":[165],"replacing":[167],"homogeneous":[173],"1.68\\%":[177],"penalty.":[178],"These":[179],"indicate":[181],"performance":[183],"depends":[184],"primarily":[185],"on":[186,194],"composition":[188],"any":[195],"particular":[196],"permutation.":[197],"confirm":[199],"finding":[201],"2.16$\\times$":[203],"larger":[204],"scale":[205],"(1.514B":[206],"parameters),":[207],"where":[208],"homogeneous-stack":[210],"penalty":[211],"increases":[212],"2.63\\%":[214],"removing":[216],"SSM-family":[218],"produces":[220],"3.20\\%":[222],"degradation.":[223],"further":[225],"report":[226],"per-mechanism":[227],"cost":[228],"profiles,":[229],"English":[230],"Korean":[232],"evaluations,":[233],"causal-safety":[236],"audit":[237],"of":[238],"all":[239],"layers.":[241],"release":[243],"weights,":[245],"training":[246,248],"recipes,":[247],"code,":[249],"logs,":[250],"architecture":[252],"source":[253],"code.":[254]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-19T00:00:00"}