{"id":"https://openalex.org/W4367367475","doi":"https://doi.org/10.48550/arxiv.2304.14204","title":"Towards Medical Artificial General Intelligence via Knowledge-Enhanced Multimodal Pretraining","display_name":"Towards Medical Artificial General Intelligence via Knowledge-Enhanced Multimodal Pretraining","publication_year":2023,"publication_date":"2023-04-26","ids":{"openalex":"https://openalex.org/W4367367475","doi":"https://doi.org/10.48550/arxiv.2304.14204"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2304.14204","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.14204","pdf_url":"https://arxiv.org/pdf/2304.14204","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2304.14204","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101923104","display_name":"Bingqian Lin","orcid":"https://orcid.org/0000-0002-8763-9530"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Lin, Bingqian","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040826162","display_name":"Zicong Chen","orcid":"https://orcid.org/0000-0002-8161-0649"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Zicong","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100416260","display_name":"Mingjie Li","orcid":"https://orcid.org/0000-0002-2989-2669"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Mingjie","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049546565","display_name":"Haokun Lin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Haokun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066768790","display_name":"Hang Xu","orcid":"https://orcid.org/0000-0003-4176-0738"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Hang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100434324","display_name":"Yi Zhu","orcid":"https://orcid.org/0000-0003-3000-3918"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039205892","display_name":"Jianzhuang Liu","orcid":"https://orcid.org/0000-0002-7960-9382"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Jianzhuang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101413407","display_name":"Wenjia Cai","orcid":"https://orcid.org/0000-0003-2398-1449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cai, Wenjia","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072565301","display_name":"Lei Yang","orcid":"https://orcid.org/0000-0002-5176-003X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Lei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100362768","display_name":"Shen Zhao","orcid":"https://orcid.org/0009-0005-2285-8555"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Shen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101571309","display_name":"Chenfei Wu","orcid":"https://orcid.org/0000-0002-5678-9691"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Chenfei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069988750","display_name":"Ling Chen","orcid":"https://orcid.org/0000-0002-6468-5729"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Ling","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034967388","display_name":"Xiaojun Chang","orcid":"https://orcid.org/0000-0002-7778-8807"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chang, Xiaojun","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005421447","display_name":"Yi Yang","orcid":"https://orcid.org/0000-0002-0512-880X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Yi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100381484","display_name":"Lei Xing","orcid":"https://orcid.org/0000-0003-2536-5359"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xing, Lei","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5047878798","display_name":"Xiaodan Liang","orcid":"https://orcid.org/0000-0003-3213-3062"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liang, Xiaodan","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":16,"corresponding_author_ids":["https://openalex.org/A5101923104"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9704999923706055,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7372449636459351},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.6911762356758118},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5691221952438354},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.540428638458252},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5320941805839539},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5313098430633545},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.49501389265060425},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.417567640542984},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39804238080978394},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.3850717842578888},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08726173639297485}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7372449636459351},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.6911762356758118},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5691221952438354},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.540428638458252},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5320941805839539},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5313098430633545},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.49501389265060425},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.417567640542984},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39804238080978394},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3850717842578888},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08726173639297485},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2304.14204","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.14204","pdf_url":"https://arxiv.org/pdf/2304.14204","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2304.14204","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2304.14204","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2304.14204","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.14204","pdf_url":"https://arxiv.org/pdf/2304.14204","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4367367475.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2905433371","https://openalex.org/W4390569940","https://openalex.org/W2888392564","https://openalex.org/W4310278675","https://openalex.org/W4388422664","https://openalex.org/W2806259446","https://openalex.org/W4361193272","https://openalex.org/W2963326959","https://openalex.org/W4312407344","https://openalex.org/W2894289927"],"abstract_inverted_index":{"Medical":[0],"artificial":[1],"general":[2,93,104],"intelligence":[3,141],"(MAGI)":[4],"enables":[5],"one":[6],"foundation":[7,111,150],"model":[8,112],"to":[9,44,62,101,152,240],"solve":[10],"different":[11,40],"medical":[12,20,37,56,90,149,157,171,188,220],"tasks,":[13,180],"which":[14,135],"is":[15],"very":[16],"practical":[17],"in":[18,97,218,257],"the":[19,26,45,103,110,131,211,219,223,232,242],"domain.":[21],"It":[22],"can":[23,117],"significantly":[24],"reduce":[25],"requirement":[27],"of":[28,31,47,88,139,142,178,226,235,244],"large":[29],"amounts":[30],"task-specific":[32,64],"data":[33,124],"by":[34],"sufficiently":[35],"sharing":[36],"knowledge":[38,116,213],"among":[39],"tasks.":[41,158],"However,":[42],"due":[43],"challenges":[46],"designing":[48],"strongly":[49],"generalizable":[50],"models":[51],"with":[52,113],"limited":[53],"and":[54,94,133,164,187],"complex":[55],"data,":[57,221],"most":[58],"existing":[59],"approaches":[60],"tend":[61],"develop":[63],"models.":[65],"To":[66,159],"take":[67],"a":[68,74,98,108,147,161,170,175,237,246,254],"step":[69],"towards":[70],"MAGI,":[71],"we":[72,84,168],"propose":[73],"new":[75],"paradigm":[76],"called":[77],"Medical-knOwledge-enhanced":[78],"mulTimOdal":[79],"pretRaining":[80],"(MOTOR).":[81],"In":[82],"MOTOR,":[83],"combine":[85],"two":[86,137],"kinds":[87,138],"basic":[89,115],"knowledge,":[91,96],"i.e.,":[92],"specific":[95],"complementary":[99],"manner":[100],"boost":[102],"pretraining":[105,122],"process.":[106],"As":[107],"result,":[109],"comprehensive":[114,162],"learn":[118],"compact":[119],"representations":[120],"from":[121],"radiographic":[123],"for":[125],"better":[126],"cross-modal":[127],"alignment.":[128],"MOTOR":[129,199,229],"unifies":[130],"understanding":[132],"generation,":[134],"are":[136],"core":[140],"an":[143],"AI":[144],"system,":[145],"into":[146],"single":[148],"model,":[151],"flexibly":[153],"handle":[154],"more":[155],"diverse":[156],"enable":[160],"evaluation":[163],"facilitate":[165],"further":[166],"research,":[167],"construct":[169],"multimodal":[172],"benchmark":[173,196],"including":[174],"wide":[176],"range":[177],"downstream":[179],"such":[181],"as":[182],"chest":[183],"x-ray":[184],"report":[185],"generation":[186],"visual":[189],"question":[190],"answering.":[191],"Extensive":[192],"experiments":[193],"on":[194],"our":[195,251],"show":[197],"that":[198,210,250],"obtains":[200],"promising":[201],"results":[202],"through":[203],"simple":[204],"task-oriented":[205],"adaptation.":[206],"The":[207],"visualization":[208],"shows":[209],"injected":[212],"successfully":[214,230],"highlights":[215],"key":[216],"information":[217],"demonstrating":[222],"excellent":[224],"interpretability":[225],"MOTOR.":[227],"Our":[228],"mimics":[231],"human":[233],"practice":[234],"fulfilling":[236],"\"medical":[238],"student\"":[239],"accelerate":[241],"process":[243],"becoming":[245],"\"specialist\".":[247],"We":[248],"believe":[249],"work":[252],"makes":[253],"significant":[255],"stride":[256],"realizing":[258],"MAGI.":[259]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3}],"updated_date":"2026-03-11T14:59:36.786465","created_date":"2023-04-30T00:00:00"}