{"id":"https://openalex.org/W4387635115","doi":"https://doi.org/10.48550/arxiv.2310.07999","title":"LEMON: Lossless model expansion","display_name":"LEMON: Lossless model expansion","publication_year":2023,"publication_date":"2023-10-12","ids":{"openalex":"https://openalex.org/W4387635115","doi":"https://doi.org/10.48550/arxiv.2310.07999"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.07999","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07999","pdf_url":"https://arxiv.org/pdf/2310.07999","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4387635115"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.07999","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077157931","display_name":"Yite Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Yite","raw_affiliation_strings":["University of Illinois Urbana-Champaign , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Illinois Urbana-Champaign , USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113006214","display_name":"Jiahao Su","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Su, Jiahao","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004170004","display_name":"Hanlin Lu","orcid":"https://orcid.org/0000-0001-6743-301X","observed_orcids":["https://orcid.org/0000-0001-6743-301X"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu, Hanlin","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016911975","display_name":"Cong Xie","orcid":"https://orcid.org/0000-0003-0062-9127","observed_orcids":["https://orcid.org/0000-0003-0062-9127"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xie, Cong","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100430615","display_name":"Tianyi Liu","orcid":"https://orcid.org/0009-0009-3747-6487","observed_orcids":["https://orcid.org/0009-0009-3747-6487"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Tianyi","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025459071","display_name":"Jianbo Yuan","orcid":"https://orcid.org/0000-0002-8905-7438","observed_orcids":["https://orcid.org/0000-0002-8905-7438"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuan, Jianbo","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110368559","display_name":"Haibin Lin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lin, Haibin","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102887924","display_name":"Ruoyu Sun","orcid":"https://orcid.org/0000-0003-0465-9083","observed_orcids":["https://orcid.org/0000-0003-0465-9083"]},"institutions":[{"id":"https://openalex.org/I4210099586","display_name":"Shenzhen Research Institute of Big Data","ror":"https://ror.org/00z1gwf89","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210099586"]},{"id":"https://openalex.org/I4210116924","display_name":"Chinese University of Hong Kong, Shenzhen","ror":"https://ror.org/02d5ks197","country_code":"CN","type":"education","lineage":["https://openalex.org/I177725633","https://openalex.org/I180726961","https://openalex.org/I4210116924"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Sun, Ruoyu","raw_affiliation_strings":["School of Data Science , The Chinese University of Hong Kong , Shenzhen , China","Shenzhen International Center for Industrial and Applied Mathematics , Shenzhen Research Institute of Big Data"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Data Science , The Chinese University of Hong Kong , Shenzhen , China","institution_ids":["https://openalex.org/I4210116924"]},{"raw_affiliation_string":"Shenzhen International Center for Industrial and Applied Mathematics , Shenzhen Research Institute of Big Data","institution_ids":["https://openalex.org/I4210099586"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082599714","display_name":"Hongxia Yang","orcid":"https://orcid.org/0000-0002-0580-9728","observed_orcids":["https://orcid.org/0000-0002-0580-9728"]},"institutions":[{"id":"https://openalex.org/I4405256866","display_name":"ByteDance (China)","ror":"https://ror.org/04m8av638","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405256866"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yang, Hongxia","raw_affiliation_strings":["ByteDance Inc"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ByteDance Inc","institution_ids":["https://openalex.org/I4405256866"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.7900000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.7900000214576721,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.19339999556541443,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.01209999993443489,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/model-extension","display_name":"model extension","score":0.7310000061988831},{"id":"https://openalex.org/keywords/knowledge-distillation","display_name":"knowledge distillation","score":0.7099999785423279},{"id":"https://openalex.org/keywords/weight-initialization","display_name":"weight initialization","score":0.4009999930858612},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.8259999752044678},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.9419999718666077},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.9980000257492065},{"id":"https://openalex.org/keywords/learning-rate-schedules","display_name":"learning rate scheduling","score":0.8379999995231628},{"id":"https://openalex.org/keywords/training-efficiency","display_name":"training efficiency","score":0.7519999742507935},{"id":"https://openalex.org/keywords/foundation-models","display_name":"foundation models","score":0.6399999856948853}],"concepts":[{"id":"https://openalex.org/C2781235140","wikidata":"https://www.wikidata.org/wiki/Q275131","display_name":"Scratch","level":2,"score":0.8332985043525696},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7352941036224365},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6713401079177856},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.6383869647979736},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5794199109077454},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.5529124736785889},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49321335554122925},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.46583476662635803},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.46179628372192383},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4517100751399994},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33312854170799255},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3229871094226837},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.20247399806976318},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.17494356632232666},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.10402140021324158},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.07227298617362976},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.07999","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07999","pdf_url":"https://arxiv.org/pdf/2310.07999","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.07999","doi":"https://doi.org/10.48550/arxiv.2310.07999","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.07999","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.07999","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.07999","pdf_url":"https://arxiv.org/pdf/2310.07999","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387635115.pdf","grobid_xml":"https://content.openalex.org/works/W4387635115.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2108598243","https://openalex.org/W2125930537","https://openalex.org/W2178031510","https://openalex.org/W2553303224","https://openalex.org/W2556833785","https://openalex.org/W2766207105","https://openalex.org/W2768501777","https://openalex.org/W2771655537","https://openalex.org/W2799054028","https://openalex.org/W2805003733","https://openalex.org/W2810075754","https://openalex.org/W2894740066","https://openalex.org/W2951598131","https://openalex.org/W2980282514","https://openalex.org/W2995492258","https://openalex.org/W2995816250","https://openalex.org/W3034290785","https://openalex.org/W3108384060","https://openalex.org/W3121924028","https://openalex.org/W3202099651","https://openalex.org/W3214576447","https://openalex.org/W4221147360","https://openalex.org/W4225364564","https://openalex.org/W4303939357","https://openalex.org/W4308900169","https://openalex.org/W4321524682","https://openalex.org/W4322718191","https://openalex.org/W4323112125","https://openalex.org/W4377089791","https://openalex.org/W4385714663","https://openalex.org/W4386942538","https://openalex.org/W4387185421","https://openalex.org/W7133245089"],"related_works":["https://openalex.org/W4412944775","https://openalex.org/W4390872547","https://openalex.org/W4323112125","https://openalex.org/W4389911345","https://openalex.org/W4416270281","https://openalex.org/W4392019598","https://openalex.org/W4403574950","https://openalex.org/W3157366283","https://openalex.org/W4404405780","https://openalex.org/W4392874056"],"abstract_inverted_index":{"Scaling":[0],"of":[1,20,74],"deep":[2],"neural":[3],"networks,":[4],"especially":[5],"Transformers,":[6],"is":[7,81,110],"pivotal":[8],"for":[9,94,136,141],"their":[10,46,75],"surging":[11],"performance":[12],"and":[13,123,139],"has":[14],"further":[15],"led":[16],"to":[17,40,53,67,104,145],"the":[18,42,72,95,100],"emergence":[19],"sophisticated":[21],"reasoning":[22],"capabilities":[23],"in":[24],"foundation":[25],"models.":[26],"Such":[27],"scaling":[28],"generally":[29],"requires":[30],"training":[31,85,101,105,146],"large":[32],"models":[33,70,119],"from":[34,106,147],"scratch":[35],"with":[36,86,114],"random":[37],"initialization,":[38],"failing":[39],"leverage":[41],"knowledge":[43],"acquired":[44],"by":[45,83,134],"smaller":[47,76],"counterparts,":[48],"which":[49],"are":[50],"already":[51],"resource-intensive":[52],"obtain.":[54],"To":[55],"tackle":[56],"this":[57],"inefficiency,":[58],"we":[59],"present":[60],"$\\textbf{L}$ossl$\\textbf{E}$ss":[61],"$\\textbf{MO}$del":[62],"Expansio$\\textbf{N}$":[63],"(LEMON),":[64],"a":[65],"recipe":[66],"initialize":[68],"scaled":[69,96],"using":[71],"weights":[73],"but":[77],"pre-trained":[78],"counterparts.":[79],"This":[80],"followed":[82],"model":[84],"an":[87],"optimized":[88],"learning":[89],"rate":[90],"scheduler":[91],"tailored":[92],"explicitly":[93],"models,":[97],"substantially":[98],"reducing":[99],"time":[102],"compared":[103,144],"scratch.":[107,148],"Notably,":[108],"LEMON":[109,130],"versatile,":[111],"ensuring":[112],"compatibility":[113],"various":[115],"network":[116],"structures,":[117],"including":[118],"like":[120],"Vision":[121,137],"Transformers":[122,138],"BERT.":[124],"Our":[125],"empirical":[126],"results":[127],"demonstrate":[128],"that":[129],"reduces":[131],"computational":[132],"costs":[133],"56.7%":[135],"33.2%":[140],"BERT":[142],"when":[143]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":11}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2023-10-14T00:00:00"}