{"id":"https://openalex.org/W7160916766","doi":"https://doi.org/10.48550/arxiv.2605.09887","title":"The Geometric Wall: Manifold Structure Predicts Layerwise Sparse Autoencoder Scaling Laws","display_name":"The Geometric Wall: Manifold Structure Predicts Layerwise Sparse Autoencoder Scaling Laws","publication_year":2026,"publication_date":"2026-05-11","ids":{"openalex":"https://openalex.org/W7160916766","doi":"https://doi.org/10.48550/arxiv.2605.09887"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.09887","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5123883865","display_name":"Eslam Zaher","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zaher, Eslam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123988146","display_name":"Maciej Trzaskowski","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Trzaskowski, Maciej","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124004815","display_name":"Quan Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Quan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5056884940","display_name":"Fred Roosta","orcid":"https://orcid.org/0000-0002-6920-7072"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Roosta, Fred","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.4154999852180481,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.4154999852180481,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.1111999973654747,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12417","display_name":"Morphological variations and asymmetry","score":0.08869999647140503,"subfield":{"id":"https://openalex.org/subfields/2608","display_name":"Geometry and Topology"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.684499979019165},{"id":"https://openalex.org/keywords/manifold","display_name":"Manifold (fluid mechanics)","score":0.649399995803833},{"id":"https://openalex.org/keywords/curvature","display_name":"Curvature","score":0.5622000098228455},{"id":"https://openalex.org/keywords/dimension","display_name":"Dimension (graph theory)","score":0.5174000263214111},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.4787999987602234},{"id":"https://openalex.org/keywords/exponent","display_name":"Exponent","score":0.4629000127315521},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.39649999141693115},{"id":"https://openalex.org/keywords/linear-model","display_name":"Linear model","score":0.3472999930381775}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6955999732017517},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.684499979019165},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.649399995803833},{"id":"https://openalex.org/C195065555","wikidata":"https://www.wikidata.org/wiki/Q214881","display_name":"Curvature","level":2,"score":0.5622000098228455},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.5174000263214111},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.4787999987602234},{"id":"https://openalex.org/C2780388253","wikidata":"https://www.wikidata.org/wiki/Q5421508","display_name":"Exponent","level":2,"score":0.4629000127315521},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.39649999141693115},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.3472999930381775},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.34619998931884766},{"id":"https://openalex.org/C124066611","wikidata":"https://www.wikidata.org/wiki/Q28684319","display_name":"Sparse approximation","level":2,"score":0.33730000257492065},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.33230000734329224},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.3264000117778778},{"id":"https://openalex.org/C16977076","wikidata":"https://www.wikidata.org/wiki/Q1589551","display_name":"Principal curvature","level":4,"score":0.298799991607666},{"id":"https://openalex.org/C30732413","wikidata":"https://www.wikidata.org/wiki/Q17092636","display_name":"Intrinsic dimension","level":3,"score":0.29319998621940613},{"id":"https://openalex.org/C104065381","wikidata":"https://www.wikidata.org/wiki/Q1002535","display_name":"Geometric modeling","level":2,"score":0.2892000079154968},{"id":"https://openalex.org/C170122806","wikidata":"https://www.wikidata.org/wiki/Q1914828","display_name":"Linear scale","level":2,"score":0.28850001096725464},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.2858999967575073},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.27559998631477356},{"id":"https://openalex.org/C151876577","wikidata":"https://www.wikidata.org/wiki/Q7049464","display_name":"Nonlinear dimensionality reduction","level":3,"score":0.26100000739097595},{"id":"https://openalex.org/C39927690","wikidata":"https://www.wikidata.org/wiki/Q11197","display_name":"Logarithm","level":2,"score":0.2590000033378601},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.2574999928474426}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.09887","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.09887","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"score":0.632715106010437,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Sparse":[0],"autoencoders":[1],"(SAEs)":[2],"operationalise":[3],"the":[4,21,61,69,90,109,144,148,164,173,182,196,206,212,216,231,260],"linear":[5,14,33,83,238],"representation":[6],"hypothesis:":[7],"they":[8,262],"reconstruct":[9],"model":[10,180],"activations":[11],"as":[12],"sparse":[13,82,237],"combinations":[15],"of":[16,64,98,128,205,240],"interpretable":[17],"dictionary":[18,84],"atoms,":[19],"on":[20,118,153,178],"implicit":[22],"assumption":[23],"that":[24,44,57,160,172,211,235],"activation":[25,70],"space":[26],"is":[27,60,72],"well":[28],"approximated":[29],"by":[30,259],"a":[31,65,95,103,137,191,241,251,255],"globally":[32],"structure.":[34],"Their":[35],"reconstruction":[36],"error":[37],"varies":[38,78],"sharply":[39],"across":[40,79,125],"layers":[41,127,198],"in":[42,168],"ways":[43],"existing":[45],"scaling":[46,93,113],"laws,":[47],"fitted":[48,145,213],"at":[49],"single":[50,104],"layers,":[51,80],"do":[52],"not":[53,250],"explain.":[54],"We":[55,107,158],"argue":[56],"this":[58],"variation":[59],"empirical":[62],"trace":[63],"geometric":[66,156,193,218],"mismatch:":[67],"where":[68,199],"manifold":[71,99,161,243,261],"curved":[73,242],"and":[74,89,116,132,147,171,222],"its":[75],"intrinsic":[76,223],"dimension":[77,224],"no":[81],"can":[85],"match":[86],"it":[87],"uniformly,":[88],"SAE's":[91],"width-sparsity":[92],"becomes":[94],"layer-dependent":[96],"function":[97],"structure":[100],"rather":[101],"than":[102],"universal":[105],"law.":[106,194],"conduct":[108],"first":[110],"cross-layer":[111],"SAE":[112,123],"study,":[114],"fitting":[115],"regressing":[117],"844":[119],"residual-stream":[120],"Gemma":[121,129],"Scope":[122],"checkpoints":[124],"68":[126],"2":[130,142],"2B":[131],"9B.":[133],"Stage":[134,141],"1":[135],"fits":[136],"per-layer":[138,150,165,185],"scaling-law":[139],"surface;":[140],"regresses":[143],"parameters":[146],"derived":[149],"width":[151,166,201],"exponents":[152,186],"four":[154],"layerwise":[155,217],"summaries.":[157],"find":[159,210],"geometry":[162],"predicts":[163],"exponent":[167],"both":[169],"models,":[170],"same":[174],"regression":[175],"coefficients":[176],"learnt":[177],"one":[179],"predict":[181],"other":[183],"model's":[184],"under":[187],"cross-model":[188],"transfer,":[189],"indicating":[190],"transferable":[192],"At":[195],"showcase":[197],"richer":[200],"grids":[202],"permit":[203],"identification":[204],"asymptotic":[207],"floor,":[208,228],"we":[209],"floor":[214],"tracks":[215],"ordering:":[219],"higher":[220,227],"curvature":[221],"correspond":[225],"to":[226,265],"consistent":[229],"with":[230],"irreducible":[232],"second-order":[233],"residual":[234],"any":[236],"approximation":[239],"must":[244],"leave":[245],"behind.":[246],"SAEs":[247],"thus":[248],"encounter":[249],"finite-resource":[252],"ceiling":[253],"but":[254],"geometry-dependent":[256],"wall,":[257],"set":[258],"are":[263],"trying":[264],"reconstruct.":[266]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2026-05-13T00:00:00"}