{"id":"https://openalex.org/W7224901722","doi":"https://doi.org/10.48550/arxiv.2610.40091","title":"Upcycling Language Models for Multiple Specialized Domains","display_name":"Upcycling Language Models for Multiple Specialized Domains","publication_year":2026,"publication_date":"2026-10-29","ids":{"openalex":"https://openalex.org/W7224901722","doi":"https://doi.org/10.48550/arxiv.2610.40091"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.40091","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40091","pdf_url":"https://arxiv.org/pdf/2610.40091","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.40091","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125583309","display_name":"Mingzi Cao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Cao, Mingzi","raw_affiliation_strings":["School of Computer Science , University of Sheffield , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science , University of Sheffield , United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5160164012","display_name":"Xi Wang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Wang, Xi","raw_affiliation_strings":["School of Computer Science , University of Sheffield , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science , University of Sheffield , United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010341007","display_name":"\u039d\u03b9\u03ba\u03cc\u03bb\u03b1\u03bf\u03c2 \u0391\u03bb\u03ad\u03c4\u03c1\u03b1\u03c2","orcid":"https://orcid.org/0000-0003-4285-1965","observed_orcids":["https://orcid.org/0000-0003-4285-1965"]},"institutions":[{"id":"https://openalex.org/I91136226","display_name":"University of Sheffield","ror":"https://ror.org/05krs5044","country_code":"GB","type":"education","lineage":["https://openalex.org/I91136226"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Aletras, Nikolaos","raw_affiliation_strings":["School of Computer Science , University of Sheffield , United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science , University of Sheffield , United Kingdom","institution_ids":["https://openalex.org/I91136226"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I91136226"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0020000000949949026,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/upcycling","display_name":"upcycling","score":0.8809999823570251},{"id":"https://openalex.org/keywords/multi-domain-adaptation","display_name":"multi-domain adaptation","score":0.8709999918937683},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9470000267028809},{"id":"https://openalex.org/keywords/expert-routing","display_name":"expert routing","score":0.5979999899864197},{"id":"https://openalex.org/keywords/parameter-efficient-fine-tuning","display_name":"parameter-efficient fine-tuning","score":0.7440000176429749},{"id":"https://openalex.org/keywords/low-rank-adaptation","display_name":"low-rank adaptation","score":0.6579999923706055}],"concepts":[{"id":"https://openalex.org/C2775896111","wikidata":"https://www.wikidata.org/wiki/Q642560","display_name":"Router","level":2,"score":0.8751000165939331},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7325999736785889},{"id":"https://openalex.org/C177284502","wikidata":"https://www.wikidata.org/wiki/Q1005390","display_name":"Adapter (computing)","level":2,"score":0.7282000184059143},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.513700008392334},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4968000054359436},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4844000041484833},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.459199994802475},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.428600013256073},{"id":"https://openalex.org/C74172769","wikidata":"https://www.wikidata.org/wiki/Q1446839","display_name":"Routing (electronic design automation)","level":2,"score":0.40790000557899475},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.3781000077724457},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.37599998712539673},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3560999929904938},{"id":"https://openalex.org/C184896649","wikidata":"https://www.wikidata.org/wiki/Q290066","display_name":"Routing table","level":4,"score":0.3407000005245209},{"id":"https://openalex.org/C2780186347","wikidata":"https://www.wikidata.org/wiki/Q11414","display_name":"Subnetwork","level":2,"score":0.32690000534057617},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.32120001316070557},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.3176000118255615},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30079999566078186},{"id":"https://openalex.org/C104954878","wikidata":"https://www.wikidata.org/wiki/Q1648707","display_name":"Routing protocol","level":3,"score":0.26579999923706055},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.2632000148296356},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.25769999623298645}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2610.40091","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40091","pdf_url":"https://arxiv.org/pdf/2610.40091","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.40091","doi":"https://doi.org/10.48550/arxiv.2610.40091","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.40091","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.40091","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40091","pdf_url":"https://arxiv.org/pdf/2610.40091","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224901722.pdf","grobid_xml":"https://content.openalex.org/works/W7224901722.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4403575587","https://openalex.org/W3215620590","https://openalex.org/W4385569751","https://openalex.org/W4226218363","https://openalex.org/W7220933331","https://openalex.org/W4401043762","https://openalex.org/W4416766941","https://openalex.org/W7091647073","https://openalex.org/W3156785025","https://openalex.org/W4386566605"],"abstract_inverted_index":{"Effective":[0],"multi-domain":[1,106],"adaptation":[2,184],"of":[3,47,59],"large":[4],"language":[5],"models":[6],"requires":[7],"prioritizing":[8],"relevant":[9],"domain":[10,32,50,174],"knowledge":[11],"while":[12,55,65,143,165],"minimizing":[13],"interference":[14],"from":[15],"irrelevant":[16,62],"domains.":[17],"Existing":[18],"approaches":[19],"commonly":[20],"use":[21],"Low-Rank":[22],"Adaptation":[23],"(LoRA)":[24],"to":[25,86,118,121,163],"train":[26],"a":[27,76,84,87,110],"separate":[28],"adapter":[29],"for":[30,105,175,188],"each":[31,132],"and":[33,83,141],"combine":[34],"these":[35],"adapters":[36,44,54],"within":[37],"one":[38],"model.":[39],"However,":[40],"they":[41],"either":[42],"weight":[43,53],"equally":[45],"regardless":[46],"the":[48,116,126,138,154,159],"input":[49],"or":[51],"dynamically":[52],"still":[56],"activating":[57],"all":[58,145],"them":[60],"including":[61],"ones.":[63],"Additionally,":[64],"efficient,":[66],"their":[67],"low-rank":[68],"structure":[69],"limits":[70],"performance.":[71,168],"In":[72,97],"contrast,":[73],"upcycling":[74],"is":[75],"model":[77],"expansion":[78],"method":[79,152],"that":[80,114,150],"adds":[81],"experts":[82],"router":[85,117,142,176],"dense":[88],"Transformer":[89],"layer":[90],"offering":[91],"larger":[92],"capacity":[93],"than":[94],"LoRA":[95],"approaches.":[96],"this":[98],"paper,":[99],"we":[100,135],"propose":[101],"Domain-Specialized":[102],"Upcycling":[103],"(DS-Upcycling)":[104],"adaptation.":[107],"We":[108],"introduce":[109],"new":[111,133],"domain-aware":[112],"supervision":[113],"allows":[115],"effectively":[119],"learn":[120],"distinguish":[122],"domains":[123,161],"by":[124],"selecting":[125],"appropriate":[127],"experts.":[128],"During":[129],"fine-tuning":[130],"on":[131],"domain,":[134],"update":[136],"only":[137],"corresponding":[139],"expert":[140],"freezing":[144],"other":[146],"parameters.":[147],"Experiments":[148],"show":[149],"our":[151,192],"achieves":[153,179],"highest":[155],"average":[156],"performance":[157],"across":[158],"adapted":[160],"compared":[162,185],"baselines":[164],"retaining":[166],"general":[167],"With":[169],"just":[170],"256":[171],"examples":[172],"per":[173],"supervision,":[177],"DS-Upcycling":[178],"90-98%":[180],"routing":[181],"accuracy":[182],"after":[183],"with":[186],"43-70%":[187],"routers":[189],"trained":[190],"without":[191],"approach.":[193]},"counts_by_year":[],"updated_date":"2026-11-02T09:45:03.842371","created_date":"2026-10-31T00:00:00"}