{"id":"https://openalex.org/W4390306224","doi":"https://doi.org/10.48550/arxiv.2312.15320","title":"GestaltMML: Enhancing Rare Genetic Disease Diagnosis through Multimodal Machine Learning Combining Facial Images and Clinical Text","display_name":"GestaltMML: Enhancing Rare Genetic Disease Diagnosis through Multimodal Machine Learning Combining Facial Images and Clinical Text","publication_year":2023,"publication_date":"2023-12-23","ids":{"openalex":"https://openalex.org/W4390306224","doi":"https://doi.org/10.48550/arxiv.2312.15320"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2312.15320","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.15320","pdf_url":"https://arxiv.org/pdf/2312.15320","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2312.15320","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101511812","display_name":"Wu Da","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Da","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113200283","display_name":"Jingye Yang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhanliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100331534","display_name":"Cong Liu","orcid":"https://orcid.org/0000-0001-6024-3037"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Hongzhuo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066913932","display_name":"Tzung\u2010Chien Hsieh","orcid":"https://orcid.org/0000-0003-3828-4419"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang, Jingye","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Cong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Cong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Hsieh, Tzung-Chien","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hsieh, Tzung-Chien","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013713722","display_name":"Peter Krawitz","orcid":"https://orcid.org/0000-0002-3194-8625"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marchi, Elaine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009604048","display_name":"Chunhua Weng","orcid":"https://orcid.org/0000-0002-9624-0214"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Blair, Justin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Krawitz, Peter","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krawitz, Peter","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009604048","display_name":"Chunhua Weng","orcid":"https://orcid.org/0000-0002-9624-0214"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weng, Chunhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chung, Wendy","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chung, Wendy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008798496","display_name":"Gholson J. Lyon","orcid":"https://orcid.org/0000-0002-5869-0716"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lyon, Gholson J.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100437036","display_name":"Kai Wang","orcid":"https://orcid.org/0000-0002-6170-4744"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Krantz, Ian D.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075046653","display_name":"Jennifer M. Kalish","orcid":"https://orcid.org/0000-0003-1500-9713"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kalish, Jennifer M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100437036","display_name":"Kai Wang","orcid":"https://orcid.org/0000-0002-6170-4744"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Kai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11928","display_name":"Genetic Syndromes and Imprinting","score":0.95169997215271,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T11928","display_name":"Genetic Syndromes and Imprinting","score":0.95169997215271,"subfield":{"id":"https://openalex.org/subfields/1311","display_name":"Genetics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5608110427856445},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5576107501983643},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5322586894035339},{"id":"https://openalex.org/keywords/exome-sequencing","display_name":"Exome sequencing","score":0.5277631282806396},{"id":"https://openalex.org/keywords/medical-diagnosis","display_name":"Medical diagnosis","score":0.48035845160484314},{"id":"https://openalex.org/keywords/genetic-architecture","display_name":"Genetic architecture","score":0.46802932024002075},{"id":"https://openalex.org/keywords/exome","display_name":"Exome","score":0.4619915783405304},{"id":"https://openalex.org/keywords/convolutional-neural-network","display_name":"Convolutional neural network","score":0.4317828416824341},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.28241994976997375},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.18012160062789917},{"id":"https://openalex.org/keywords/phenotype","display_name":"Phenotype","score":0.15552273392677307},{"id":"https://openalex.org/keywords/genetics","display_name":"Genetics","score":0.141029953956604},{"id":"https://openalex.org/keywords/pathology","display_name":"Pathology","score":0.1243995726108551}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5608110427856445},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5576107501983643},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5322586894035339},{"id":"https://openalex.org/C16671776","wikidata":"https://www.wikidata.org/wiki/Q5420592","display_name":"Exome sequencing","level":4,"score":0.5277631282806396},{"id":"https://openalex.org/C534262118","wikidata":"https://www.wikidata.org/wiki/Q177719","display_name":"Medical diagnosis","level":2,"score":0.48035845160484314},{"id":"https://openalex.org/C9287583","wikidata":"https://www.wikidata.org/wiki/Q5532883","display_name":"Genetic architecture","level":4,"score":0.46802932024002075},{"id":"https://openalex.org/C10590036","wikidata":"https://www.wikidata.org/wiki/Q417664","display_name":"Exome","level":5,"score":0.4619915783405304},{"id":"https://openalex.org/C81363708","wikidata":"https://www.wikidata.org/wiki/Q17084460","display_name":"Convolutional neural network","level":2,"score":0.4317828416824341},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.28241994976997375},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.18012160062789917},{"id":"https://openalex.org/C127716648","wikidata":"https://www.wikidata.org/wiki/Q104053","display_name":"Phenotype","level":3,"score":0.15552273392677307},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.141029953956604},{"id":"https://openalex.org/C142724271","wikidata":"https://www.wikidata.org/wiki/Q7208","display_name":"Pathology","level":1,"score":0.1243995726108551},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2312.15320","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.15320","pdf_url":"https://arxiv.org/pdf/2312.15320","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"pmh:oai:arXiv.org:2312.15320","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2312.15320","pdf_url":"https://arxiv.org/pdf/2312.15320","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2312.15320","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2312.15320","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2312.15320","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2312.15320","pdf_url":"https://arxiv.org/pdf/2312.15320","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2093932542","https://openalex.org/W2043256480","https://openalex.org/W2172231381","https://openalex.org/W1983406990","https://openalex.org/W2106924416","https://openalex.org/W2772505697","https://openalex.org/W3083501552","https://openalex.org/W2400191746","https://openalex.org/W2484239316","https://openalex.org/W3024198590"],"abstract_inverted_index":{"Individuals":[0],"with":[1],"suspected":[2],"rare":[3,43,230],"genetic":[4,16,44,68],"disorders":[5],"often":[6],"undergo":[7],"multiple":[8],"clinical":[9,58,141],"evaluations,":[10],"imaging":[11],"studies,":[12],"laboratory":[13,66],"tests,":[14],"and":[15,39,70,101,140,165,169,184,205,212,232],"tests":[17],"over":[18],"a":[19,24,120,144],"prolonged":[20],"period":[21],"of":[22,76,146,229,237],"time,":[23],"process":[25],"commonly":[26],"described":[27],"as":[28],"the":[29,73,128,162,190,196,226,235],"diagnostic":[30,197],"odyssey.":[31],"Addressing":[32],"this":[33],"odyssey":[34],"has":[35],"substantial":[36],"clinical,":[37],"psychosocial,":[38],"economic":[40],"benefits.":[41],"Many":[42],"diseases":[45,63,160,231],"have":[46],"distinctive":[47],"facial":[48,87,99,133],"features":[49],"that":[50,84,110,221],"artificial":[51],"intelligence":[52],"algorithms":[53],"can":[54,223],"use":[55,85],"to":[56,60,71,151],"facilitate":[57,234],"diagnosis,":[59],"prioritize":[61],"candidate":[62,227],"for":[64,113,200],"further":[65],"or":[67,78,107],"testing,":[69],"support":[72],"phenotype-driven":[74],"reinterpretation":[75,236],"genome":[77],"exome":[79],"sequencing":[80,238],"data.":[81,239],"Existing":[82],"methods":[83],"frontal":[86],"photographs":[88],"were":[89],"built":[90],"on":[91,98,127,158,166,189],"conventional":[92],"convolutional":[93],"neural":[94],"networks,":[95],"rely":[96],"exclusively":[97],"images,":[100,134],"cannot":[102],"capture":[103],"non-facial":[104],"phenotypic":[105],"traits":[106],"demographic":[108,135],"information":[109,136],"are":[111],"essential":[112],"accurate":[114],"diagnosis.":[115],"Here":[116],"we":[117],"introduce":[118],"GestaltMML,":[119],"multimodal":[121,208],"machine":[122],"learning":[123],"approach":[124],"based":[125],"solely":[126],"Transformer":[129],"architecture.":[130],"It":[131],"integrates":[132],"(age,":[137],"sex,":[138],"ethnicity),":[139],"notes":[142],"(optionally":[143],"list":[145],"Human":[147],"Phenotype":[148],"Ontology":[149],"terms)":[150],"improve":[152],"prediction":[153],"accuracy.":[154],"We":[155],"evaluate":[156],"GestaltMML":[157,187,222],"528":[159],"from":[161,202],"GestaltMatcher":[163],"Database":[164],"several":[167],"in-house":[168],"published":[170],"cohorts,":[171],"including":[172],"Beckwith-Wiedemann":[173],"syndrome,":[174,176,179,183],"Sotos":[175],"NAA10-related":[177],"neurodevelopmental":[178],"Cornelia":[180],"de":[181],"Lange":[182],"KBG":[185],"syndrome.":[186],"improves":[188],"state-of-the-art":[191],"image-only":[192,214],"ensembled":[193],"model,":[194],"narrows":[195],"accuracy":[198],"gap":[199],"patients":[201],"under-represented":[203],"ancestries,":[204],"clarifies":[206],"when":[207,213],"fusion":[209],"is":[210,216],"beneficial":[211],"inference":[215],"preferable.":[217],"The":[218],"results":[219],"suggest":[220],"greatly":[224],"narrow":[225],"diagnoses":[228],"may":[233]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-18T07:39:51.176621","created_date":"2023-12-29T00:00:00"}