{"id":"https://openalex.org/W7155226100","doi":"https://doi.org/10.48550/arxiv.2604.19394","title":"Can Continual Pre-training Bridge the Performance Gap between General-purpose and Specialized Language Models in the Medical Domain?","display_name":"Can Continual Pre-training Bridge the Performance Gap between General-purpose and Specialized Language Models in the Medical Domain?","publication_year":2026,"publication_date":"2026-04-21","ids":{"openalex":"https://openalex.org/W7155226100","doi":"https://doi.org/10.48550/arxiv.2604.19394"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.19394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.19394","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010015333","display_name":"Niclas Doll","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Doll, Niclas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093073362","display_name":"Jasper Schulze Buschhoff","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Buschhoff, Jasper Schulze","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002333681","display_name":"Shalaka Satheesh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Satheesh, Shalaka","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071912952","display_name":"Hammam Abdelwahab","orcid":"https://orcid.org/0009-0000-6283-2310"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abdelwahab, Hammam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043402158","display_name":"H\u00e9ctor Allende\u2010Cid","orcid":"https://orcid.org/0000-0003-3047-8817"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Allende-Cid, H\u00e9ctor","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5109771934","display_name":"Katrin Klug","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Klug, Katrin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.1898999959230423,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.1898999959230423,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.17000000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.15549999475479126,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bridging","display_name":"Bridging (networking)","score":0.6618000268936157},{"id":"https://openalex.org/keywords/merge","display_name":"Merge (version control)","score":0.5461999773979187},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.545199990272522},{"id":"https://openalex.org/keywords/scarcity","display_name":"Scarcity","score":0.5149000287055969},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4943000078201294},{"id":"https://openalex.org/keywords/german","display_name":"German","score":0.4909000098705292},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4163999855518341},{"id":"https://openalex.org/keywords/domain-adaptation","display_name":"Domain adaptation","score":0.4043000042438507}],"concepts":[{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.6618000268936157},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6312000155448914},{"id":"https://openalex.org/C197129107","wikidata":"https://www.wikidata.org/wiki/Q1921621","display_name":"Merge (version control)","level":2,"score":0.5461999773979187},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.545199990272522},{"id":"https://openalex.org/C109747225","wikidata":"https://www.wikidata.org/wiki/Q815758","display_name":"Scarcity","level":2,"score":0.5149000287055969},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4943000078201294},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.4909000098705292},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4163999855518341},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.4043000042438507},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.37950000166893005},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.3573000133037567},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.34220001101493835},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.3336000144481659},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.32760000228881836},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3010999858379364},{"id":"https://openalex.org/C45493050","wikidata":"https://www.wikidata.org/wiki/Q7884934","display_name":"Unified Model","level":2,"score":0.2879999876022339},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2831000089645386},{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.27790001034736633},{"id":"https://openalex.org/C92548554","wikidata":"https://www.wikidata.org/wiki/Q2262868","display_name":"Domain model","level":3,"score":0.273499995470047},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.27149999141693115},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2623000144958496},{"id":"https://openalex.org/C3018838755","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Medical care","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C135257023","wikidata":"https://www.wikidata.org/wiki/Q691358","display_name":"Domain-specific language","level":2,"score":0.25619998574256897}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.19394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.19394","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.19394","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0,41,104,154],"paper":[1],"narrows":[2],"the":[3,25,60,80,85,94,97,136,145,167],"performance":[4,74],"gap":[5],"between":[6],"small,":[7],"specialized":[8,28,107,163],"models":[9,14,87,109],"and":[10,21,48,141],"significantly":[11],"larger":[12,99],"general-purpose":[13],"through":[15,101],"domain":[16,102],"adaptation":[17],"via":[18],"continual":[19],"pre-training":[20],"merging.":[22],"We":[23],"address":[24],"scarcity":[26],"of":[27,84,138],"non-English":[29],"data":[30],"by":[31],"constructing":[32],"a":[33,111,127,157],"high-quality":[34],"German":[35,76],"medical":[36,77,117],"corpus":[37,42],"(FineMed-de)":[38],"from":[39,54],"FineWeb2.":[40],"is":[43],"used":[44],"to":[45,56],"continually":[46],"pre-train":[47],"merge":[49],"three":[50],"well-known":[51],"LLMs":[52],"(ranging":[53],"$7B$":[55,72,108],"$24B$":[57],"parameters),":[58],"creating":[59],"DeFineMed":[61],"model":[62,73,121],"family.":[63],"A":[64],"comprehensive":[65],"evaluation":[66],"confirms":[67],"that":[68],"specialization":[69],"dramatically":[70],"enhances":[71],"on":[75],"benchmarks.":[78],"Furthermore,":[79],"pairwise":[81],"win-rate":[82,95],"analysis":[83,131],"Qwen2.5-based":[86],"demonstrates":[88],"an":[89],"approximately":[90],"$3.5$-fold":[91],"increase":[92],"in":[93,151,172],"against":[96],"much":[98],"Mistral-Small-24B-Instruct":[100],"adaptation.":[103],"evidence":[105],"positions":[106],"as":[110,166],"competitive,":[112],"resource-efficient":[113],"solution":[114],"for":[115,147,161,169],"complex":[116],"instruction-following":[118,125],"tasks.":[119],"While":[120],"merging":[122],"successfully":[123],"restores":[124],"abilities,":[126],"subsequent":[128],"failure":[129],"mode":[130],"reveals":[132],"inherent":[133],"trade-offs,":[134],"including":[135],"introduction":[137],"language":[139],"mixing":[140],"increased":[142],"verbosity,":[143],"highlighting":[144],"need":[146],"more":[148],"targeted":[149],"fine-tuning":[150],"future":[152],"work.":[153],"research":[155],"provides":[156],"robust,":[158],"compliant":[159],"methodology":[160],"developing":[162],"LLMs,":[164],"serving":[165],"foundation":[168],"practical":[170],"use":[171],"German-speaking":[173],"healthcare":[174],"contexts.":[175]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-23T00:00:00"}