{"id":"https://openalex.org/W4416249141","doi":"https://doi.org/10.1101/2025.11.10.25339903","title":"Leveraging simulation to provide a practical framework for assessing the novel scope of risk of LLMs in healthcare","display_name":"Leveraging simulation to provide a practical framework for assessing the novel scope of risk of LLMs in healthcare","publication_year":2025,"publication_date":"2025-11-13","ids":{"openalex":"https://openalex.org/W4416249141","doi":"https://doi.org/10.1101/2025.11.10.25339903"},"language":"en","primary_location":{"id":"doi:10.1101/2025.11.10.25339903","doi":"https://doi.org/10.1101/2025.11.10.25339903","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.11.10.25339903","pdf_url":"https://www.medrxiv.org/content/medrxiv/early/2025/11/13/2025.11.10.25339903.full.pdf","source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"volume":null,"type":"preprint","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.medrxiv.org/content/medrxiv/early/2025/11/13/2025.11.10.25339903.full.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027636442","display_name":"Mark Kalinich","orcid":"https://orcid.org/0000-0002-8794-1289","observed_orcids":["https://orcid.org/0000-0002-8794-1289"]},"institutions":[{"id":"https://openalex.org/I1316535847","display_name":"Beth Israel Deaconess Medical Center","ror":"https://ror.org/04drvxt59","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1316535847"]},{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]},{"id":"https://openalex.org/I2801337628","display_name":"Beth Israel Deaconess Hospital","ror":"https://ror.org/02cpjkp59","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I2801337628"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Mark Kalinich","raw_affiliation_strings":["Harvard Medical School, BIDMC Department of Psychiatry"],"raw_orcid":"https://orcid.org/0000-0002-8794-1289","affiliations":[{"raw_affiliation_string":"Harvard Medical School, BIDMC Department of Psychiatry","institution_ids":["https://openalex.org/I1316535847","https://openalex.org/I136199984","https://openalex.org/I2801337628"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004493831","display_name":"James Luccarelli","orcid":"https://orcid.org/0000-0002-1606-3917","observed_orcids":["https://orcid.org/0000-0002-1606-3917"]},"institutions":[{"id":"https://openalex.org/I4210087915","display_name":"Massachusetts General Hospital","ror":"https://ror.org/002pd6e78","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210087915","https://openalex.org/I48633490"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James Luccarelli","raw_affiliation_strings":["Massachusetts General Hospital"],"raw_orcid":"https://orcid.org/0000-0002-1606-3917","affiliations":[{"raw_affiliation_string":"Massachusetts General Hospital","institution_ids":["https://openalex.org/I4210087915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Frank Moss","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frank Moss","raw_affiliation_strings":["Strategic Software Ventures, LLC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Strategic Software Ventures, LLC","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5026523515","display_name":"John Blake Torous","orcid":"https://orcid.org/0000-0002-5362-7937","observed_orcids":["https://orcid.org/0000-0002-5362-7937"]},"institutions":[{"id":"https://openalex.org/I1316535847","display_name":"Beth Israel Deaconess Medical Center","ror":"https://ror.org/04drvxt59","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I1316535847"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Torous","raw_affiliation_strings":["BIDMC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"BIDMC","institution_ids":["https://openalex.org/I1316535847"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5027636442"],"corresponding_institution_ids":["https://openalex.org/I1316535847","https://openalex.org/I136199984","https://openalex.org/I2801337628"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9487000107765198,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9487000107765198,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.019300000742077827,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13295","display_name":"Safety Systems Engineering in Autonomy","score":0.016599999740719795,"subfield":{"id":"https://openalex.org/subfields/2213","display_name":"Safety, Risk, Reliability and Quality"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9810000061988831},{"id":"https://openalex.org/keywords/medical-device-regulation","display_name":"medical device regulation","score":0.6660000085830688},{"id":"https://openalex.org/keywords/risk-assessment","display_name":"risk assessment","score":0.675000011920929},{"id":"https://openalex.org/keywords/suicidal-ideation-detection","display_name":"suicidal ideation detection","score":0.9129999876022339},{"id":"https://openalex.org/keywords/chatbot-safety","display_name":"chatbot safety","score":0.4580000042915344},{"id":"https://openalex.org/keywords/artificial-intelligence-in-clinical-practice","display_name":"clinical implementation of AI","score":0.6079999804496765},{"id":"https://openalex.org/keywords/false-negative-rate","display_name":"false negative rate","score":0.5870000123977661},{"id":"https://openalex.org/keywords/uncertainty-quantification","display_name":"uncertainty quantification","score":0.38499999046325684}],"concepts":[{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.626800000667572},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.6104999780654907},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.5629000067710876},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5266000032424927},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5198000073432922},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5192000269889832},{"id":"https://openalex.org/C22507642","wikidata":"https://www.wikidata.org/wiki/Q1069369","display_name":"Hazardous waste","level":2,"score":0.47290000319480896},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.4616999924182892},{"id":"https://openalex.org/C73722673","wikidata":"https://www.wikidata.org/wiki/Q1082833","display_name":"Probabilistic risk assessment","level":3,"score":0.427700012922287},{"id":"https://openalex.org/C49261128","wikidata":"https://www.wikidata.org/wiki/Q1132455","display_name":"Hazard","level":2,"score":0.42480000853538513},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.3564999997615814},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3068000078201294},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C32896092","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Risk management","level":2,"score":0.26969999074935913},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.2685000002384186},{"id":"https://openalex.org/C2779208394","wikidata":"https://www.wikidata.org/wiki/Q7140460","display_name":"Participatory sensing","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.262800008058548},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2533000111579895},{"id":"https://openalex.org/C45804977","wikidata":"https://www.wikidata.org/wiki/Q7239673","display_name":"Predictive modelling","level":2,"score":0.2522999942302704}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1101/2025.11.10.25339903","doi":"https://doi.org/10.1101/2025.11.10.25339903","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.11.10.25339903","pdf_url":"https://www.medrxiv.org/content/medrxiv/early/2025/11/13/2025.11.10.25339903.full.pdf","source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.1101/2025.11.10.25339903","doi":"https://doi.org/10.1101/2025.11.10.25339903","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2025.11.10.25339903","pdf_url":"https://www.medrxiv.org/content/medrxiv/early/2025/11/13/2025.11.10.25339903.full.pdf","source":{"id":"https://openalex.org/S3005729997","display_name":"medRxiv","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":["https://openalex.org/P4310315909"],"host_organization_lineage_names":["Cold Spring Harbor Laboratory Press"],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"medRxiv","raw_type":"posted-content"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","score":0.9537000060081482,"display_name":"Good health and well-being"}],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416249141.pdf","grobid_xml":"https://content.openalex.org/works/W4416249141.grobid-xml"},"referenced_works_count":4,"referenced_works":["https://openalex.org/W4283170666","https://openalex.org/W4406152291","https://openalex.org/W4410343193","https://openalex.org/W4410745976"],"related_works":["https://openalex.org/W7165759053","https://openalex.org/W7196962902","https://openalex.org/W7154491711","https://openalex.org/W7164208877","https://openalex.org/W7131671601","https://openalex.org/W7158888898","https://openalex.org/W7214115591","https://openalex.org/W7166304247","https://openalex.org/W7154688201","https://openalex.org/W7166695580"],"abstract_inverted_index":{"Structured":[0],"Abstract":[1],"Background":[2],"Large":[3],"language":[4,281,364],"models":[5,116,218,365],"(LLMs)":[6,366],"are":[7],"rapidly":[8],"entering":[9],"clinical":[10,46,341,416],"care,":[11],"yet":[12],"their":[13],"definitionally":[14],"probabilistic":[15,160],"outputs":[16],"have":[17],"delivered":[18],"a":[19,52,75,78,84,89,104,169,173,290,309,352],"variety":[20],"of":[21,48,99,162,168,182,266,279,343,388],"grossly":[22],"unsafe":[23],"responses":[24],"to":[25,40,68,121,172,237,244,252,257,263,274,302,338,377,406],"users.":[26],"The":[27],"difficulty":[28],"in":[29,186,211,368],"quantifying":[30,294],"and":[31,45,77,91,127,140,152,176,205,213,247,298,323,327,357,384,399,408,414],"mitigating":[32,358],"the":[33,42,96,166,180,276,340,359,372,386,394],"novel":[34,97,277],"risks":[35,98,278,361],"posed":[36],"by":[37,148,154,198],"LLMs":[38],"threatens":[39],"stall":[41],"regulatory":[43,286,316,381],"evaluation":[44],"deployment":[47],"LLM-based":[49,344],"software":[50],"as":[51],"medical":[53,345],"device":[54],"(LLM-SaMD).":[55],"A":[56],"practical,":[57],"evidence-based":[58],"framework":[59,86,105,354,395],"is":[60,404],"urgently":[61],"needed":[62,405],"for":[63,94,203,293,321,355],"extending":[64],"existing":[65,271,380],"medical-device":[66],"regulations":[67],"encompass":[69],"LLM-SaMDs.":[70,100],"Using":[71],"synthetic":[72,193],"interactions":[73],"between":[74],"chatbot":[76],"potentially":[79],"suicidal":[80],"user,":[81],"we":[82],"demonstrate":[83],"simulation-based":[85],"that":[87,183,312,330,362],"provides":[88,289],"reproducible":[90,291],"generalizable":[92,310],"method":[93,292],"evaluating":[95,356],"Methods":[101],"We":[102],"developed":[103],"integrating":[106],"LLM":[107,189],"performance":[108,202,223,411],"testing":[109],"into":[110],"SaMD":[111],"risk":[112],"estimation.":[113],"Fourteen":[114],"open-source":[115],"ranging":[117],"from":[118,241,254,374],"270":[119],"million":[120],"70":[122],"billion":[123,221],"parameters":[124],"(Qwen,":[125],"Gemma,":[126],"LLaMA":[128],"families)":[129],"were":[130,146],"evaluated":[131],"on":[132],"three":[133],"safety-classification":[134],"tasks:":[135],"suicidal-ideation":[136,212],"detection,":[137,139],"therapy-request":[138],"therapy-like":[141,214],"interaction":[142],"detection.":[143],"Synthetic":[144],"datasets":[145,195],"generated":[147],"Gemini":[149],"2.5":[150],"Pro":[151],"verified":[153],"psychiatrists.":[155],"Model":[156],"false-negative":[157],"rates":[158],"informed":[159],"estimates":[161],"P":[163,177,233,248],"1":[164,234],",":[165,179,260],"likelihood":[167,181],"hazard":[170],"progressing":[171],"hazardous":[174,238],"situation,":[175],"2":[178,249],"situation":[184,251],"resulting":[185],"harm.":[187],"Results":[188],"success":[190],"at":[191],"generating":[192],"safety":[194],"varied":[196],"substantially":[197],"task,":[199],"with":[200,226],"strong":[201],"neutral":[204],"non-therapeutic":[206],"content":[207],"but":[208,228],"frequent":[209],"errors":[210],"interactions.":[215],"Across":[216],"14":[217],"(270":[219],"million\u201370":[220],"parameters),":[222],"generally":[224],"improved":[225],"size":[227],"included":[229],"notable":[230],"outliers.":[231],"Estimated":[232],"values":[235],"(hazard":[236],"situation)":[239],"ranged":[240],"2.0\u00d710":[242],"-8":[243],"2.6\u00d710":[245],"-4":[246],"(hazardous":[250],"harm)":[253],"7.1\u00d710":[255],"-5":[256],"9.6\u00d710":[258],"-3":[259],"spanning":[261],"up":[262],"four":[264],"orders":[265],"magnitude.":[267],"Conclusion":[268],"Simulation":[269],"extends":[270],"device-safety":[272],"frameworks":[273],"address":[275],"large":[280,363],"models.":[282],"Rather":[283],"than":[284],"replacing":[285],"judgment,":[287],"it":[288],"uncertainty,":[295],"clarifying":[296],"assumptions,":[297],"linking":[299],"model":[300,410],"failures":[301],"plausible":[303],"harms.":[304],"Our":[305],"case":[306],"example":[307],"demonstrates":[308],"approach":[311],"can":[313],"overcome":[314],"current":[315],"barriers":[317,337],"while":[318,334],"remaining":[319],"practical":[320],"manufacturers":[322],"regulators,":[324],"supporting":[325],"timely":[326],"transparent":[328],"oversight":[329],"keeps":[331],"patients":[332],"safe":[333],"avoiding":[335],"unnecessary":[336],"delivering":[339],"promise":[342],"devices.":[346],"Brief":[347],"Description":[348],"This":[349],"study":[350],"introduces":[351],"quantitative":[353],"unique":[360],"pose":[367],"healthcare.":[369],"By":[370],"mapping":[371],"pathways":[373],"LLM-generated":[375],"hazards":[376],"harms":[378],"onto":[379],"risk-analysis":[382],"structures":[383],"estimating":[385],"probability":[387],"these":[389],"transitions":[390],"through":[391],"computational":[392],"simulation,":[393],"empirically":[396],"bounds":[397],"uncertainty":[398],"identifies":[400],"where":[401],"real-world":[402],"evidence":[403],"validate":[407],"monitor":[409],"before,":[412],"during,":[413],"after":[415],"deployment.":[417]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-10-30T12:39:25.321706","created_date":"2025-11-14T00:00:00"}