{"id":"https://openalex.org/W4291988272","doi":"https://doi.org/10.48550/arxiv.2208.06399","title":"AutoShard: Automated Embedding Table Sharding for Recommender Systems","display_name":"AutoShard: Automated Embedding Table Sharding for Recommender Systems","publication_year":2022,"publication_date":"2022-08-12","ids":{"openalex":"https://openalex.org/W4291988272","doi":"https://doi.org/10.48550/arxiv.2208.06399"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2208.06399","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.06399","pdf_url":"https://arxiv.org/pdf/2208.06399","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2208.06399","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058071176","display_name":"Daochen Zha","orcid":"https://orcid.org/0000-0002-6677-7504"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zha, Daochen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100808706","display_name":"Louis Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Louis","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079877271","display_name":"Bhargav Bhushanam","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bhushanam, Bhargav","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046749324","display_name":"Dhruv Choudhary","orcid":"https://orcid.org/0000-0002-4520-765X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choudhary, Dhruv","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004002767","display_name":"Jade Nie","orcid":"https://orcid.org/0009-0008-1664-2805"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nie, Jade","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106668475","display_name":"Yuandong Tian","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian, Yuandong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086705017","display_name":"Jay Chae","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chae, Jay","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078483147","display_name":"Yinbin Ma","orcid":"https://orcid.org/0000-0002-9511-6596"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Yinbin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047988079","display_name":"Arun Kejariwal","orcid":"https://orcid.org/0009-0006-6172-2973"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kejariwal, Arun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5068477431","display_name":"Xia Hu","orcid":"https://orcid.org/0000-0003-2234-3226"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Xia","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5058071176"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11980","display_name":"Human Mobility and Location-Based Analysis","score":0.9883999824523926,"subfield":{"id":"https://openalex.org/subfields/3313","display_name":"Transportation"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10331","display_name":"Video Surveillance and Tracking Methods","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.8599864840507507},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8136810660362244},{"id":"https://openalex.org/keywords/partition","display_name":"Partition (number theory)","score":0.7266313433647156},{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.6785380840301514},{"id":"https://openalex.org/keywords/table","display_name":"Table (database)","score":0.5569747686386108},{"id":"https://openalex.org/keywords/transferability","display_name":"Transferability","score":0.48016753792762756},{"id":"https://openalex.org/keywords/categorical-variable","display_name":"Categorical variable","score":0.4360889196395874},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.38604485988616943},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3254814147949219},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.26596778631210327},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09673157334327698}],"concepts":[{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.8599864840507507},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8136810660362244},{"id":"https://openalex.org/C42812","wikidata":"https://www.wikidata.org/wiki/Q1082910","display_name":"Partition (number theory)","level":2,"score":0.7266313433647156},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.6785380840301514},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.5569747686386108},{"id":"https://openalex.org/C61272859","wikidata":"https://www.wikidata.org/wiki/Q7834031","display_name":"Transferability","level":3,"score":0.48016753792762756},{"id":"https://openalex.org/C5274069","wikidata":"https://www.wikidata.org/wiki/Q2285707","display_name":"Categorical variable","level":2,"score":0.4360889196395874},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38604485988616943},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3254814147949219},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.26596778631210327},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09673157334327698},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C140331021","wikidata":"https://www.wikidata.org/wiki/Q1868104","display_name":"Logit","level":2,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2208.06399","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.06399","pdf_url":"https://arxiv.org/pdf/2208.06399","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"doi:10.48550/arxiv.2208.06399","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2208.06399","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2208.06399","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2208.06399","pdf_url":"https://arxiv.org/pdf/2208.06399","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4291988272.pdf","grobid_xml":"https://content.openalex.org/works/W4291988272.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2161221533","https://openalex.org/W4229699405","https://openalex.org/W1666484574","https://openalex.org/W2216382288","https://openalex.org/W2355491300","https://openalex.org/W4234629551","https://openalex.org/W2011110943","https://openalex.org/W2028856635","https://openalex.org/W2011433332","https://openalex.org/W2582594227"],"abstract_inverted_index":{"Embedding":[0],"learning":[1,145],"is":[2,64,91,98,112,231],"an":[3,23,154],"important":[4],"technique":[5],"in":[6,125,205,225],"deep":[7,143],"recommendation":[8],"models":[9],"to":[10,14,42,57,84,100,114,135,146,179],"map":[11],"categorical":[12],"features":[13],"dense":[15],"vectors.":[16],"However,":[17,50],"the":[18,31,44,51,81,86,105,109,138,148,164,169,172,191],"embedding":[19,45,52,73,82],"tables":[20,46,53,83,186,193,204],"often":[21],"demand":[22],"extremely":[24],"large":[25],"number":[26],"of":[27,69,166,175,185,190,203,212],"parameters,":[28],"which":[29,90,129],"become":[30],"storage":[32],"and":[33,102,107,141,159,187,210],"efficiency":[34,211],"bottlenecks.":[35],"Distributed":[36],"training":[37],"solutions":[38],"have":[39,222],"been":[40,223],"adopted":[41],"partition":[43,80,110,149],"into":[47],"multiple":[48],"devices.":[49],"can":[54,177,199],"easily":[55],"lead":[56],"imbalances":[58],"if":[59],"not":[60],"carefully":[61],"partitioned.":[62],"This":[63],"a":[65,92,131],"significant":[66],"design":[67],"challenge":[68],"distributed":[70],"systems":[71],"named":[72],"table":[74],"sharding,":[75],"i.e.,":[76],"how":[77],"we":[78,120],"should":[79],"balance":[85],"costs":[87,140],"across":[88],"devices,":[89],"non-trivial":[93],"task":[94],"because":[95],"1)":[96],"it":[97,215],"hard":[99],"efficiently":[101,200],"precisely":[103],"measure":[104],"cost,":[106],"2)":[108],"problem":[111],"known":[113],"be":[115],"NP-hard.":[116],"In":[117],"this":[118],"work,":[119],"introduce":[121],"our":[122],"novel":[123],"practice":[124],"Meta,":[126],"namely":[127],"AutoShard,":[128],"uses":[130],"neural":[132],"cost":[133],"model":[134],"directly":[136],"predict":[137],"multi-table":[139],"leverages":[142],"reinforcement":[144],"solve":[147],"problem.":[150],"Experimental":[151],"results":[152],"on":[153],"open-sourced":[155],"large-scale":[156],"synthetic":[157],"dataset":[158,162],"Meta's":[160],"production":[161,218,227],"demonstrate":[163],"superiority":[165],"AutoShard":[167,176,198,213],"over":[168],"heuristics.":[170],"Moreover,":[171],"learned":[173],"policy":[174],"transfer":[178],"sharding":[180],"tasks":[181],"with":[182],"various":[183],"numbers":[184],"different":[188],"ratios":[189],"unseen":[192],"without":[194],"any":[195],"fine-tuning.":[196],"Furthermore,":[197],"shard":[201],"hundreds":[202],"seconds.":[206],"The":[207],"effectiveness,":[208],"transferability,":[209],"make":[214],"desirable":[216],"for":[217],"use.":[219],"Our":[220],"algorithms":[221],"deployed":[224],"Meta":[226],"environment.":[228],"A":[229],"prototype":[230],"available":[232],"at":[233],"https://github.com/daochenzha/autoshard":[234]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}