{"id":"https://openalex.org/W4285077484","doi":"https://doi.org/10.48550/arxiv.2207.04648","title":"Learning Large-scale Universal User Representation with Sparse Mixture of Experts","display_name":"Learning Large-scale Universal User Representation with Sparse Mixture of Experts","publication_year":2022,"publication_date":"2022-07-11","ids":{"openalex":"https://openalex.org/W4285077484","doi":"https://doi.org/10.48550/arxiv.2207.04648"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2207.04648","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.04648","pdf_url":"https://arxiv.org/pdf/2207.04648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2207.04648","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045901919","display_name":"Caigao Jiang","orcid":"https://orcid.org/0000-0001-9383-2479","observed_orcids":["https://orcid.org/0000-0001-9383-2479"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang, Caigao","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086364554","display_name":"Siqiao Xue","orcid":"https://orcid.org/0000-0002-0291-6536","observed_orcids":["https://orcid.org/0000-0002-0291-6536"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xue, Siqiao","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101545814","display_name":"James Zhang","orcid":"https://orcid.org/0000-0002-1867-7875","observed_orcids":["https://orcid.org/0000-0002-1867-7875","https://orcid.org/0000-0001-6519-676X"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, James","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102794276","display_name":"Lingyue Liu","orcid":"https://orcid.org/0000-0002-5000-1605","observed_orcids":["https://orcid.org/0000-0002-5000-1605"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Lingyue","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101997766","display_name":"Zhibo Zhu","orcid":"https://orcid.org/0000-0003-0241-4007","observed_orcids":["https://orcid.org/0000-0003-0241-4007","https://orcid.org/0009-0005-0185-212X"]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhu, Zhibo","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5112303454","display_name":"Hongyan Hao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405270328","display_name":"Ant Group (China)","ror":"https://ror.org/00g5w7n47","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405270328"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao, Hongyan","raw_affiliation_strings":["Ant Group , Hangzhou , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ant Group , Hangzhou , China","institution_ids":["https://openalex.org/I4405270328"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4405270328"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9434000253677368,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9434000253677368,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.029100000858306885,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/user-representation-learning","display_name":"user representation learning","score":0.7429999709129333},{"id":"https://openalex.org/keywords/sparse-mixture-of-experts","display_name":"sparse mixture-of-experts","score":0.8960000276565552},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"multi-task learning","score":0.8169999718666077}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7524139285087585},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.661732017993927},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.5799111127853394},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.5373820662498474},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.45437878370285034},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44094622135162354},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4385029375553131},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.43534162640571594},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.35649874806404114},{"id":"https://openalex.org/C89505385","wikidata":"https://www.wikidata.org/wiki/Q47146","display_name":"User interface","level":2,"score":0.2849176526069641},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08552271127700806},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2207.04648","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.04648","pdf_url":"https://arxiv.org/pdf/2207.04648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2207.04648","doi":"https://doi.org/10.48550/arxiv.2207.04648","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2207.04648","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article-journal"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2207.04648","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2207.04648","pdf_url":"https://arxiv.org/pdf/2207.04648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4285077484.pdf","grobid_xml":"https://content.openalex.org/works/W4285077484.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W2896457183","https://openalex.org/W3094502228","https://openalex.org/W3100417358","https://openalex.org/W3119866685","https://openalex.org/W3195577433","https://openalex.org/W3201103413","https://openalex.org/W4226012237","https://openalex.org/W4296591867","https://openalex.org/W4382239917","https://openalex.org/W4386408441"],"related_works":["https://openalex.org/W3210761106","https://openalex.org/W4294719260","https://openalex.org/W3130666114","https://openalex.org/W4413928280","https://openalex.org/W7107859397","https://openalex.org/W4413928318","https://openalex.org/W3177361862","https://openalex.org/W4382239917","https://openalex.org/W4387849061","https://openalex.org/W3167826227"],"abstract_inverted_index":{"Learning":[0],"user":[1,22,54,60,73,79,111,118],"sequence":[2],"behaviour":[3,55,119],"embedding":[4,61,74],"is":[5],"very":[6],"sophisticated":[7],"and":[8,18,30,87,126,173,190],"challenging":[9],"due":[10],"to":[11,39,107,134,140,146],"the":[12,51,88,95,117,131,184,191,194],"complicated":[13],"feature":[14],"interactions":[15],"over":[16,187],"time":[17],"high":[19,109],"dimensions":[20],"of":[21,37,53,75,136,142,196],"features.":[23],"Recent":[24],"emerging":[25],"foundation":[26],"models,":[27,189],"e.g.,":[28],"BERT":[29],"its":[31],"variants,":[32],"encourage":[33],"a":[34,71,104,158],"large":[35,76],"body":[36],"researchers":[38],"investigate":[40],"in":[41,69],"this":[42,99],"field.":[43],"However,":[44],"unlike":[45],"natural":[46],"language":[47],"processing":[48],"(NLP)":[49],"tasks,":[50,86,155],"parameters":[52],"model":[56,132],"come":[57],"mostly":[58],"from":[59,83,113],"layer,":[62],"which":[63],"makes":[64],"most":[65],"existing":[66],"works":[67],"fail":[68],"training":[70],"universal":[72],"scale.":[77],"Furthermore,":[78],"representations":[80],"are":[81,121],"learned":[82],"multiple":[84,114,154],"downstream":[85],"past":[89],"research":[90],"work":[91],"do":[92],"not":[93],"address":[94],"seesaw":[96,149],"phenomenon.":[97],"In":[98,144],"paper,":[100],"we":[101,127,156],"propose":[102],"SUPERMOE,":[103],"generic":[105],"framework":[106],"obtain":[108],"quality":[110],"representation":[112],"tasks.":[115],"Specifically,":[116],"sequences":[120],"encoded":[122],"by":[123],"MoE":[124],"transformer,":[125],"can":[128],"thus":[129],"increase":[130],"capacity":[133],"billions":[135],"parameters,":[137],"or":[138],"even":[139],"trillions":[141],"parameters.":[143],"order":[145],"deal":[147],"with":[148,162],"phenomenon":[150],"when":[151],"learning":[152],"across":[153],"design":[157],"new":[159],"loss":[160],"function":[161],"task":[163],"indicators.":[164],"We":[165],"perform":[166],"extensive":[167],"offline":[168],"experiments":[169,175],"on":[170,176],"public":[171],"datasets":[172],"online":[174],"private":[177],"real-world":[178],"business":[179],"scenarios.":[180],"Our":[181],"approach":[182],"achieves":[183],"best":[185],"performance":[186],"state-of-the-art":[188],"results":[192],"demonstrate":[193],"effectiveness":[195],"our":[197],"framework.":[198]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2022-07-13T00:00:00"}