{"id":"https://openalex.org/W7079548401","doi":"https://doi.org/10.48550/arxiv.2509.02565","title":"Understanding sparse autoencoder scaling in the presence of feature manifolds","display_name":"Understanding sparse autoencoder scaling in the presence of feature manifolds","alternate_titles":[],"publication_year":2025,"publication_date":"2025-09-02","ids":{"openalex":"https://openalex.org/W7079548401","doi":"https://doi.org/10.48550/arxiv.2509.02565"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2509.02565","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02565","pdf_url":"https://arxiv.org/pdf/2509.02565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7079548401"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.02565","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071048251","display_name":"Eric J. Michaud","orcid":"https://orcid.org/0000-0001-7912-1953","observed_orcids":["https://orcid.org/0000-0001-7912-1953"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michaud, Eric J.","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Gorton, Liv","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gorton, Liv","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035693801","display_name":"Tom McGrath","orcid":"https://orcid.org/0000-0001-5873-1621","observed_orcids":["https://orcid.org/0000-0001-5873-1621"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"McGrath, Tom","raw_affiliation_strings":["Massachusetts Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8999000191688538,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.8999000191688538,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.030799999833106995,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.01360000018030405,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sparse-autoencoders","display_name":"sparse autoencoders","score":0.996999979019165},{"id":"https://openalex.org/keywords/neural-scaling-laws","display_name":"neural scaling laws","score":0.7680000066757202},{"id":"https://openalex.org/keywords/feature-manifold","display_name":"feature manifold","score":0.8610000014305115},{"id":"https://openalex.org/keywords/capacity-allocation","display_name":"capacity allocation","score":0.6399999856948853},{"id":"https://openalex.org/keywords/latent-dimensionality","display_name":"latent dimensionality","score":0.6050000190734863}],"concepts":[{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.7975999712944031},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.7720999717712402},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6420000195503235},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5893999934196472},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.5737000107765198},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5254999995231628},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4799000024795532},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.367900013923645},{"id":"https://openalex.org/C91682802","wikidata":"https://www.wikidata.org/wiki/Q620538","display_name":"Multidimensional scaling","level":2,"score":0.36000001430511475},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.337799996137619},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3154999911785126},{"id":"https://openalex.org/C83665646","wikidata":"https://www.wikidata.org/wiki/Q42139305","display_name":"Feature vector","level":2,"score":0.29589998722076416},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2888000011444092},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2782999873161316},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2988430800","wikidata":"https://www.wikidata.org/wiki/Q428971","display_name":"Scaling law","level":3,"score":0.2596000134944916},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.25429999828338623},{"id":"https://openalex.org/C2944601119","wikidata":"https://www.wikidata.org/wiki/Q43744058","display_name":"Residual neural network","level":3,"score":0.2542000114917755},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.2542000114917755}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2509.02565","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02565","pdf_url":"https://arxiv.org/pdf/2509.02565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.02565","doi":"https://doi.org/10.48550/arxiv.2509.02565","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.02565","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.02565","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2509.02565","pdf_url":"https://arxiv.org/pdf/2509.02565","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G539115378","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"2141064","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7079548401.pdf","grobid_xml":"https://content.openalex.org/works/W7079548401.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W3001279689","https://openalex.org/W4296932880","https://openalex.org/W4386839891","https://openalex.org/W4398888709","https://openalex.org/W4399455238","https://openalex.org/W4401200976","https://openalex.org/W4402502788","https://openalex.org/W4403851170","https://openalex.org/W4403996123","https://openalex.org/W4404344174","https://openalex.org/W4404752287","https://openalex.org/W4405254162","https://openalex.org/W4405300917","https://openalex.org/W4407759842","https://openalex.org/W4414580992","https://openalex.org/W4414895261","https://openalex.org/W4415085995","https://openalex.org/W4415135936","https://openalex.org/W4415798500","https://openalex.org/W6948385943","https://openalex.org/W7133225605","https://openalex.org/W7133228893"],"related_works":["https://openalex.org/W7160916766","https://openalex.org/W7159661641","https://openalex.org/W4399455238","https://openalex.org/W4417095740","https://openalex.org/W7222708030","https://openalex.org/W7160951579","https://openalex.org/W4403365499","https://openalex.org/W7167276265","https://openalex.org/W4415190559","https://openalex.org/W7129268635"],"abstract_inverted_index":{"Sparse":[0],"autoencoders":[1],"(SAEs)":[2],"model":[3,42,72],"the":[4,31,44,71,84,102,120],"activations":[5,26],"of":[6,13,17,22,33,87],"a":[7,40],"neural":[8,45],"network":[9],"as":[10],"linear":[11],"combinations":[12],"sparsely":[14],"occurring":[15],"directions":[16],"variation":[18],"(latents).":[19],"The":[20],"ability":[21],"SAEs":[23,89,113],"to":[24,50,57,90],"reconstruct":[25],"follows":[27],"scaling":[28,46,65,75],"laws":[29],"w.r.t.":[30],"number":[32],"latents.":[34],"In":[35],"this":[36,116],"work,":[37,70],"we":[38],"adapt":[39],"capacity-allocation":[41],"from":[43],"literature":[47],"(Brill,":[48],"2024)":[49],"understand":[51,58],"SAE":[52],"scaling,":[53],"and":[54],"in":[55,78,95,101,115,119],"particular,":[56],"how":[59],"\"feature":[60],"manifolds\"":[61],"(multi-dimensional":[62],"features)":[63],"influence":[64],"behavior.":[66],"Consistent":[67],"with":[68],"prior":[69],"recovers":[73],"distinct":[74],"regimes.":[76],"Notably,":[77],"one":[79],"regime,":[80],"feature":[81],"manifolds":[82],"have":[83],"pathological":[85,117],"effect":[86],"causing":[88],"learn":[91],"far":[92],"fewer":[93],"features":[94],"data":[96],"than":[97],"there":[98],"are":[99,114],"latents":[100],"SAE.":[103],"We":[104],"provide":[105],"some":[106],"preliminary":[107],"discussion":[108],"on":[109],"whether":[110],"or":[111],"not":[112],"regime":[118],"wild.":[121]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7079548401","counts_by_year":[{"year":2026,"cited_by_count":10}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2025-10-10T00:00:00"}
