{"id":"https://openalex.org/W4400065274","doi":"https://doi.org/10.48550/arxiv.2406.17748","title":"A New Perspective on Shampoo's Preconditioner","display_name":"A New Perspective on Shampoo's Preconditioner","publication_year":2024,"publication_date":"2024-06-25","ids":{"openalex":"https://openalex.org/W4400065274","doi":"https://doi.org/10.48550/arxiv.2406.17748"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2406.17748","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17748","pdf_url":"https://arxiv.org/pdf/2406.17748","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.17748","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086429101","display_name":"Depen Morwani","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Morwani, Depen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018766326","display_name":"Itai Shapira","orcid":"https://orcid.org/0000-0003-1635-8464"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shapira, Itai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102775428","display_name":"Nikhil Vyas","orcid":"https://orcid.org/0000-0002-4055-7693"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vyas, Nikhil","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069983576","display_name":"Eran Malach","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Malach, Eran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108381794","display_name":"Sham M. Kakade","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kakade, Sham","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5099582937","display_name":"Lucas Janson","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Janson, Lucas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5086429101"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10690","display_name":"Vibration Control and Rheological Fluids","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10690","display_name":"Vibration Control and Rheological Fluids","score":0.7045999765396118,"subfield":{"id":"https://openalex.org/subfields/2205","display_name":"Civil and Structural Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10822","display_name":"Acoustic Wave Phenomena Research","score":0.6388999819755554,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/shampoo","display_name":"Shampoo","score":0.8126240968704224},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.6710536479949951},{"id":"https://openalex.org/keywords/preconditioner","display_name":"Preconditioner","score":0.4668028652667999},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.353149950504303},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.17699435353279114},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.1704200804233551},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.10066619515419006},{"id":"https://openalex.org/keywords/organic-chemistry","display_name":"Organic chemistry","score":0.062023043632507324}],"concepts":[{"id":"https://openalex.org/C2781171460","wikidata":"https://www.wikidata.org/wiki/Q180204","display_name":"Shampoo","level":2,"score":0.8126240968704224},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.6710536479949951},{"id":"https://openalex.org/C167431342","wikidata":"https://www.wikidata.org/wiki/Q1754327","display_name":"Preconditioner","level":3,"score":0.4668028652667999},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.353149950504303},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.17699435353279114},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.1704200804233551},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.10066619515419006},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.062023043632507324},{"id":"https://openalex.org/C159694833","wikidata":"https://www.wikidata.org/wiki/Q2321565","display_name":"Iterative method","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2406.17748","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17748","pdf_url":"https://arxiv.org/pdf/2406.17748","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2406.17748","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.17748","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2406.17748","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.17748","pdf_url":"https://arxiv.org/pdf/2406.17748","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1264789558","display_name":null,"funder_award_id":"W911NF2010021","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G3521671702","display_name":null,"funder_award_id":"2134157","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4720003262","display_name":null,"funder_award_id":"N00014-22","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G5224039946","display_name":null,"funder_award_id":"DE-SC0022199","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G5877837315","display_name":null,"funder_award_id":"DE-SC0022199","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"},{"id":"https://openalex.org/G826810440","display_name":null,"funder_award_id":"2229881","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4400065274.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2364209800","https://openalex.org/W1521615856","https://openalex.org/W4390921525","https://openalex.org/W2554164948","https://openalex.org/W2262030258","https://openalex.org/W2351848437","https://openalex.org/W4365796709","https://openalex.org/W2388685859"],"abstract_inverted_index":{"Shampoo,":[0],"a":[1,7,75,98,115],"second-order":[2],"optimization":[3],"algorithm":[4,105],"which":[5],"uses":[6],"Kronecker":[8,60,111,131],"product":[9,61,112,132],"preconditioner,":[10],"has":[11],"recently":[12],"garnered":[13],"increasing":[14],"attention":[15],"from":[16],"the":[17,34,38,41,45,58,67,85,88,92,102,108,129,136,143,158,162,166],"machine":[18],"learning":[19],"community.":[20],"The":[21],"preconditioner":[22],"used":[23,90],"by":[24,48,70,91],"Shampoo":[25,93,151],"can":[26],"be":[27],"viewed":[28],"either":[29],"as":[30,156],"an":[31,52],"approximation":[32,62,68,89,138],"of":[33,37,44,63,87,101,117,145,168],"Gauss--Newton":[35],"component":[36],"Hessian":[39,137,169],"or":[40],"covariance":[42],"matrix":[43],"gradients":[46],"maintained":[47],"Adagrad.":[49],"We":[50],"provide":[51],"explicit":[53],"and":[54,66,119,161],"novel":[55],"connection":[56,73],"between":[57],"$\\textit{optimal}$":[59],"these":[64],"matrices":[65],"made":[69],"Shampoo.":[71],"Our":[72],"highlights":[74],"subtle":[76],"but":[77],"common":[78],"misconception":[79],"about":[80],"Shampoo's":[81],"approximation.":[82,113,133,170],"In":[83],"particular,":[84],"$\\textit{square}$":[86],"optimizer":[94],"is":[95,126],"equivalent":[96],"to":[97,128,149],"single":[99],"step":[100],"power":[103],"iteration":[104],"for":[106,135],"computing":[107],"aforementioned":[109],"optimal":[110,130],"Across":[114],"variety":[116],"datasets":[118],"architectures":[120],"we":[121,140],"empirically":[122,141],"demonstrate":[123],"that":[124],"this":[125],"close":[127],"Additionally,":[134],"viewpoint,":[139],"study":[142],"impact":[144],"various":[146],"practical":[147],"tricks":[148],"make":[150],"more":[152],"computationally":[153],"efficient":[154],"(such":[155],"using":[157],"batch":[159],"gradient":[160],"empirical":[163],"Fisher)":[164],"on":[165],"quality":[167]},"counts_by_year":[],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}