{"id":"https://openalex.org/W4313197536","doi":"https://doi.org/10.48550/arxiv.2212.13138","title":"Large Language Models Encode Clinical Knowledge","display_name":"Large Language Models Encode Clinical Knowledge","publication_year":2022,"publication_date":"2022-12-26","ids":{"openalex":"https://openalex.org/W4313197536","doi":"https://doi.org/10.48550/arxiv.2212.13138"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2212.13138","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.13138","pdf_url":"https://arxiv.org/pdf/2212.13138","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2212.13138","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5027454515","display_name":"Karan Singhal","orcid":"https://orcid.org/0000-0001-9002-7490"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singhal, Karan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047463591","display_name":"Shekoofeh Azizi","orcid":"https://orcid.org/0000-0002-7447-6031"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Azizi, Shekoofeh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059213795","display_name":"Tao Tu","orcid":"https://orcid.org/0000-0003-3420-7889"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tu, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063201022","display_name":"S. Sara Mahdavi","orcid":"https://orcid.org/0000-0001-6823-598X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mahdavi, S. Sara","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wei, Jason","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Jason","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051828575","display_name":"Hyung Won Chung","orcid":"https://orcid.org/0000-0002-1280-9953"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chung, Hyung Won","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030765685","display_name":"Nathan Scales","orcid":"https://orcid.org/0000-0002-9535-7138"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scales, Nathan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088063475","display_name":"Ajay Kumar Tanwani","orcid":"https://orcid.org/0000-0002-6365-8315"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tanwani, Ajay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069557194","display_name":"Heather Cole-Lewis","orcid":"https://orcid.org/0000-0002-7275-1810"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cole-Lewis, Heather","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021812637","display_name":"Stephen Pfohl","orcid":"https://orcid.org/0000-0003-0551-9664"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pfohl, Stephen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014637990","display_name":"Perry W. Payne","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Payne, Perry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058677067","display_name":"Martin Seneviratne","orcid":"https://orcid.org/0000-0003-0435-3738"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Seneviratne, Martin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090718376","display_name":"Paul Gamble","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gamble, Paul","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061380147","display_name":"Christopher B. Kelly","orcid":"https://orcid.org/0000-0002-5530-8606"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kelly, Chris","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019198296","display_name":"Nathaneal Scharli","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Scharli, Nathaneal","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055969617","display_name":"Aakanksha Chowdhery","orcid":"https://orcid.org/0000-0002-0628-5225"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chowdhery, Aakanksha","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086361722","display_name":"P. Mansfield","orcid":"https://orcid.org/0000-0003-4969-0543"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mansfield, Philip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044698998","display_name":"Blaise Ag\u00fcera y Arcas","orcid":"https://orcid.org/0000-0003-2256-9823"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Arcas, Blaise Aguera y","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109223020","display_name":"Dale A. Webster","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Webster, Dale","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068955381","display_name":"Greg S. Corrado","orcid":"https://orcid.org/0000-0001-8817-0992"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Corrado, Greg S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065128060","display_name":"Yossi Matias","orcid":"https://orcid.org/0000-0003-3960-6002"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matias, Yossi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070366042","display_name":"Katherine Chou","orcid":"https://orcid.org/0000-0002-0318-7857"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chou, Katherine","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057932939","display_name":"Juraj Gottweis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gottweis, Juraj","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057195145","display_name":"Nenad Toma\u0161ev","orcid":"https://orcid.org/0000-0003-1624-0220"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tomasev, Nenad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078784976","display_name":"Yun Liu","orcid":"https://orcid.org/0000-0003-4079-8275"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022388476","display_name":"Alvin Rajkomar","orcid":"https://orcid.org/0000-0001-5750-5016"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rajkomar, Alvin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043862316","display_name":"Jo\u00eblle Barral","orcid":"https://orcid.org/0009-0009-0432-5148"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barral, Joelle","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010171106","display_name":"Christopher Semturs","orcid":"https://orcid.org/0000-0001-6108-2773"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Semturs, Christopher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003509342","display_name":"Alan Karthikesalingam","orcid":"https://orcid.org/0000-0001-5074-898X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Karthikesalingam, Alan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5103234563","display_name":"Vivek Natarajan","orcid":"https://orcid.org/0000-0001-7849-2074"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Natarajan, Vivek","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":262,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9793000221252441,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9097999930381775,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.701022744178772},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.630707323551178},{"id":"https://openalex.org/keywords/harm","display_name":"Harm","score":0.521195113658905},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5076378583908081},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4579000771045685},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4505102336406708},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.42464545369148254},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.41275736689567566},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.181361585855484},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.101512610912323}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.701022744178772},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.630707323551178},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.521195113658905},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5076378583908081},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4579000771045685},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4505102336406708},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.42464545369148254},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41275736689567566},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.181361585855484},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.101512610912323},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2212.13138","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.13138","pdf_url":"https://arxiv.org/pdf/2212.13138","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:RePEc:nat:nature:v:620:y:2023:i:7972:d:10.1038_s41586-023-06291-2","is_oa":false,"landing_page_url":"https://www.nature.com/articles/s41586-023-06291-2","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"doi:10.48550/arxiv.2212.13138","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2212.13138","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2212.13138","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.13138","pdf_url":"https://arxiv.org/pdf/2212.13138","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7300000190734863}],"awards":[],"funders":[{"id":"https://openalex.org/F4320317153","display_name":"DeepMind","ror":"https://ror.org/00971b260"},{"id":"https://openalex.org/F4320337372","display_name":"U.S. National Library of Medicine","ror":"https://ror.org/0060t0j89"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Large":[0],"language":[1,10],"models":[2,253],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"capabilities":[7],"in":[8,168,227,248],"natural":[9],"understanding":[11],"and":[12,20,48,75,78,107,118,210,217,245],"generation,":[13],"but":[14,198],"the":[15,222,239],"quality":[16],"bar":[17],"for":[18,93,182,254],"medical":[19,72,85,211],"clinical":[21,30,144,255],"applications":[22],"is":[23,41],"high.":[24],"Today,":[25],"attempts":[26],"to":[27,44,185,201],"assess":[28],"models'":[29],"knowledge":[31],"typically":[32],"rely":[33],"on":[34,37,123,135,149],"automated":[35],"evaluations":[36,231],"limited":[38],"benchmarks.":[39],"There":[40],"no":[42],"standard":[43],"evaluate":[45,112],"model":[46,97,215],"predictions":[47],"reasoning":[49,212],"across":[50],"a":[51,61,80,91,126,179,189],"breadth":[52],"of":[53,84,96,128,208,225,235,241],"tasks.":[54],"To":[55,171],"address":[56],"this,":[57],"we":[58,111,174],"present":[59],"MultiMedQA,":[60],"benchmark":[62],"combining":[63],"six":[64],"existing":[65],"open":[66],"question":[67],"answering":[68],"datasets":[69],"spanning":[70],"professional":[71],"exams,":[73],"research,":[74],"consumer":[76],"queries;":[77],"HealthSearchQA,":[79],"new":[81,186],"free-response":[82],"dataset":[83,139],"questions":[86],"searched":[87],"online.":[88],"We":[89,203],"propose":[90],"framework":[92],"human":[94,163,230],"evaluation":[95,164,243],"answers":[98],"along":[99],"multiple":[100],"axes":[101],"including":[102,146],"factuality,":[103],"precision,":[104],"possible":[105],"harm,":[106],"bias.":[108],"In":[109],"addition,":[110],"PaLM":[113],"(a":[114],"540-billion":[115],"parameter":[116],"LLM)":[117],"its":[119],"instruction-tuned":[120],"variant,":[121],"Flan-PaLM,":[122],"MultiMedQA.":[124],"Using":[125],"combination":[127],"prompting":[129],"strategies,":[130],"Flan-PaLM":[131,169],"achieves":[132],"state-of-the-art":[133,158],"accuracy":[134,148],"every":[136],"MultiMedQA":[137],"multiple-choice":[138],"(MedQA,":[140],"MedMCQA,":[141],"PubMedQA,":[142],"MMLU":[143],"topics),":[145],"67.6%":[147],"MedQA":[150],"(US":[151],"Medical":[152],"License":[153],"Exam":[154],"questions),":[155],"surpassing":[156],"prior":[157],"by":[159],"over":[160],"17%.":[161],"However,":[162],"reveals":[165],"key":[166],"gaps":[167],"responses.":[170],"resolve":[172],"this":[173],"introduce":[175],"instruction":[176,218],"prompt":[177,219],"tuning,":[178,220],"parameter-efficient":[180],"approach":[181],"aligning":[183],"LLMs":[184,226],"domains":[187],"using":[188],"few":[190],"exemplars.":[191],"The":[192],"resulting":[193],"model,":[194],"Med-PaLM,":[195],"performs":[196],"encouragingly,":[197],"remains":[199],"inferior":[200],"clinicians.":[202],"show":[204],"that":[205],"comprehension,":[206],"recall":[207],"knowledge,":[209],"improve":[213],"with":[214],"scale":[216],"suggesting":[221],"potential":[223],"utility":[224],"medicine.":[228],"Our":[229],"reveal":[232],"important":[233],"limitations":[234],"today's":[236],"models,":[237],"reinforcing":[238],"importance":[240],"both":[242],"frameworks":[244],"method":[246],"development":[247],"creating":[249],"safe,":[250],"helpful":[251],"LLM":[252],"applications.":[256]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":58},{"year":2024,"cited_by_count":113},{"year":2023,"cited_by_count":85}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2025-10-10T00:00:00"}