{"id":"https://openalex.org/W3206952036","doi":"https://doi.org/10.48550/arxiv.2110.04844","title":"Frequency-aware SGD for Efficient Embedding Learning with Provable Benefits","display_name":"Frequency-aware SGD for Efficient Embedding Learning with Provable Benefits","publication_year":2021,"publication_date":"2021-10-10","ids":{"openalex":"https://openalex.org/W3206952036","doi":"https://doi.org/10.48550/arxiv.2110.04844","mag":"3206952036"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2110.04844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.04844","pdf_url":"https://arxiv.org/pdf/2110.04844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.04844","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100443879","display_name":"Yan Li","orcid":"https://orcid.org/0000-0002-4694-4926","observed_orcids":["https://orcid.org/0000-0002-4694-4926","https://orcid.org/0000-0002-1126-9772","https://orcid.org/0009-0008-2313-9327"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Li, Yan","raw_affiliation_strings":["Georgia Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046749324","display_name":"Dhruv Choudhary","orcid":"https://orcid.org/0000-0002-4520-765X","observed_orcids":["https://orcid.org/0000-0002-4520-765X"]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Choudhary, Dhruv","raw_affiliation_strings":["Facebook"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101806339","display_name":"Xiaohan Wei","orcid":"https://orcid.org/0000-0001-9997-0469","observed_orcids":["https://orcid.org/0000-0001-9997-0469","https://orcid.org/0000-0001-8602-9346"]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Wei, Xiaohan","raw_affiliation_strings":["Facebook"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048995526","display_name":"Baichuan Yuan","orcid":"https://orcid.org/0000-0003-3408-2983","observed_orcids":["https://orcid.org/0000-0003-3408-2983"]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Yuan, Baichuan","raw_affiliation_strings":["Facebook"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079877271","display_name":"Bhargav Bhushanam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210111288","display_name":"Meta (United Kingdom)","ror":"https://ror.org/020ye1821","country_code":"GB","type":"company","lineage":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]},{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Bhushanam, Bhargav","raw_affiliation_strings":["Facebook"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook","institution_ids":["https://openalex.org/I4210111288","https://openalex.org/I4210114444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101595500","display_name":"Tuo Zhao","orcid":"https://orcid.org/0000-0002-1536-7302","observed_orcids":["https://orcid.org/0000-0002-1536-7302"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhao, Tuo","raw_affiliation_strings":["Georgia Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5042544900","display_name":"Guanghui Lan","orcid":"https://orcid.org/0000-0002-2103-087X","observed_orcids":["https://orcid.org/0000-0002-2103-087X"]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lan, Guanghui","raw_affiliation_strings":["Georgia Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.006899999920278788,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding-learning","display_name":"embedding learning","score":0.8519999980926514},{"id":"https://openalex.org/keywords/frequency-information","display_name":"frequency information","score":0.6169999837875366},{"id":"https://openalex.org/keywords/convergence-rate","display_name":"convergence rate","score":0.46700000762939453},{"id":"https://openalex.org/keywords/nonconvex-optimization","display_name":"non-convex optimization","score":0.906000018119812},{"id":"https://openalex.org/keywords/recommender-systems","display_name":"recommender systems","score":0.8880000114440918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.77737957239151},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6335249543190002},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.60794597864151},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.6063432097434998},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5918165445327759},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5877700448036194},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4980459213256836},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.4573097229003906},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.4496975541114807},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4120176136493683},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3434574604034424},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3355868458747864},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.1672551929950714},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2110.04844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.04844","pdf_url":"https://arxiv.org/pdf/2110.04844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2110.04844","doi":"https://doi.org/10.48550/arxiv.2110.04844","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2110.04844","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.04844","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.04844","pdf_url":"https://arxiv.org/pdf/2110.04844","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Quality education","id":"https://metadata.un.org/sdg/4","score":0.5}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1614298861","https://openalex.org/W1975768153","https://openalex.org/W2041258965","https://openalex.org/W2048176942","https://openalex.org/W2076618162","https://openalex.org/W2095984592","https://openalex.org/W2113802117","https://openalex.org/W2114827335","https://openalex.org/W2117686388","https://openalex.org/W2133029037","https://openalex.org/W2146502635","https://openalex.org/W2153579005","https://openalex.org/W2170265466","https://openalex.org/W2194775991","https://openalex.org/W2250539671","https://openalex.org/W2295739661","https://openalex.org/W2525332836","https://openalex.org/W2604662567","https://openalex.org/W2622263826","https://openalex.org/W2723293840","https://openalex.org/W2769133055","https://openalex.org/W2785523195","https://openalex.org/W2793784276","https://openalex.org/W2805767638","https://openalex.org/W2886463271","https://openalex.org/W2886837499","https://openalex.org/W2891952073","https://openalex.org/W2951489508","https://openalex.org/W2963341956","https://openalex.org/W2963470657","https://openalex.org/W2965373594","https://openalex.org/W2992505801","https://openalex.org/W2993258424","https://openalex.org/W3009948090","https://openalex.org/W3032501247","https://openalex.org/W3035161196","https://openalex.org/W3096312061","https://openalex.org/W3098608318","https://openalex.org/W3103334733","https://openalex.org/W3103362336","https://openalex.org/W3125645198","https://openalex.org/W3129751290"],"related_works":["https://openalex.org/W17155033","https://openalex.org/W3207760230","https://openalex.org/W1496222301","https://openalex.org/W1590307681","https://openalex.org/W4388717445","https://openalex.org/W2964170259","https://openalex.org/W3002546633","https://openalex.org/W4206119629","https://openalex.org/W2765682467","https://openalex.org/W4382937879"],"abstract_inverted_index":{"Embedding":[0],"learning":[1,22,37,48,63,97,156,164],"has":[2],"found":[3],"widespread":[4],"applications":[5],"in":[6,60],"recommendation":[7,131,137],"systems":[8],"and":[9,70,102,133,145],"natural":[10],"language":[11],"modeling,":[12],"among":[13],"other":[14],"domains.":[15],"To":[16],"learn":[17],"quality":[18],"embeddings":[19],"efficiently,":[20],"adaptive":[21,74,127,146],"rate":[23,49,98,157],"algorithms":[24,75,120,128],"have":[25],"demonstrated":[26],"superior":[27],"empirical":[28],"performance":[29,141],"over":[30],"SGD,":[31],"largely":[32],"accredited":[33],"to":[34,66,81,107,123,153],"their":[35],"token-dependent":[36,47,155],"rate.":[38],"However,":[39],"the":[40,44,61,78,110,118,140,151],"underlying":[41],"mechanism":[42],"for":[43,99,161],"efficiency":[45],"of":[46,58],"remains":[50],"underexplored.":[51],"We":[52],"show":[53,117,154],"that":[54,72],"incorporating":[55],"frequency":[56,79],"information":[57,80],"tokens":[59],"embedding":[62,163],"problems":[64],"leads":[65],"provably":[67,158],"efficient":[68],"algorithms,":[69],"demonstrate":[71],"common":[73],"implicitly":[76],"exploit":[77],"a":[82,95,134],"large":[83],"extent.":[84],"Specifically,":[85],"we":[86,116],"propose":[87],"(Counter-based)":[88],"Frequency-aware":[89],"Stochastic":[90],"Gradient":[91],"Descent,":[92],"which":[93],"applies":[94],"frequency-dependent":[96],"each":[100],"token,":[101],"exhibits":[103],"provable":[104],"speed-up":[105],"compared":[106],"SGD":[108,144],"when":[109],"token":[111],"distribution":[112],"is":[113],"imbalanced.":[114],"Empirically,":[115],"proposed":[119],"are":[121,150],"able":[122],"improve":[124],"or":[125],"match":[126],"on":[129],"benchmark":[130],"tasks":[132],"large-scale":[135],"industrial":[136],"system,":[138],"closing":[139],"gap":[142],"between":[143],"algorithms.":[147],"Our":[148],"results":[149],"first":[152],"improves":[159],"convergence":[160],"non-convex":[162],"problems.":[165]},"counts_by_year":[],"updated_date":"2026-10-12T09:40:04.860316","created_date":"2025-10-10T00:00:00"}