{"id":"https://openalex.org/W4391901165","doi":"https://doi.org/10.48550/arxiv.2402.10083","title":"Fine-tuning Large Language Model (LLM) Artificial Intelligence Chatbots in Ophthalmology and LLM-based evaluation using GPT-4","display_name":"Fine-tuning Large Language Model (LLM) Artificial Intelligence Chatbots in Ophthalmology and LLM-based evaluation using GPT-4","publication_year":2024,"publication_date":"2024-02-15","ids":{"openalex":"https://openalex.org/W4391901165","doi":"https://doi.org/10.48550/arxiv.2402.10083"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.10083","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.10083","pdf_url":"https://arxiv.org/pdf/2402.10083","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.10083","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5105159188","display_name":"Ting Fang Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Tan, Ting Fang","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073732346","display_name":"Kabilan Elangovan","orcid":"https://orcid.org/0000-0002-7711-7368"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Elangovan, Kabilan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101303176","display_name":"Liyuan Jin","orcid":"https://orcid.org/0000-0001-6738-1424"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Liyuan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104146191","display_name":"Jie Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jie, Yao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024710279","display_name":"Yong Li","orcid":"https://orcid.org/0000-0002-8949-8612"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yong, Li","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051944984","display_name":"Joshua Lim","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lim, Joshua","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087349472","display_name":"Stanley Poh","orcid":"https://orcid.org/0000-0003-1695-0468"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Poh, Stanley","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019123752","display_name":"Wei Yan Ng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ng, Wei Yan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034189842","display_name":"Daniel V. Lim","orcid":"https://orcid.org/0000-0001-6492-1755"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lim, Daniel","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102019719","display_name":"Yuhe Ke","orcid":"https://orcid.org/0000-0001-7193-4749"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ke, Yuhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100367554","display_name":"Nan Liu","orcid":"https://orcid.org/0000-0003-3610-4883"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Nan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5032625133","display_name":"Daniel Shu Wei Ting","orcid":"https://orcid.org/0000-0003-2264-7174"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ting, Daniel Shu Wei","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":12,"corresponding_author_ids":["https://openalex.org/A5105159188"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5238451957702637},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5159741044044495}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5238451957702637},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5159741044044495}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.10083","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.10083","pdf_url":"https://arxiv.org/pdf/2402.10083","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.10083","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.10083","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.10083","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.10083","pdf_url":"https://arxiv.org/pdf/2402.10083","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391901165.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2478288626","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W1596801655"],"abstract_inverted_index":{"Purpose:":[0],"To":[1],"assess":[2],"the":[3,13,64,77,130,146,185,192,200,216,228,240],"alignment":[4,207],"of":[5,15,108,165,208,219,242],"GPT-4-based":[6],"evaluation":[7,14,91,111,150,210,218,237],"to":[8,17,37,76,95,214,223],"human":[9,155],"clinician":[10,156],"experts,":[11],"for":[12,86,120,248],"responses":[16,75,222],"ophthalmology-related":[18],"patient":[19,41,104],"queries":[20],"generated":[21,81],"by":[22,35,82,117,134,199],"fine-tuned":[23,84,126],"LLM":[24,220,246],"chatbots.":[25],"Methods:":[26],"400":[27],"ophthalmology":[28],"questions":[29],"and":[30,48,61,106,141,160,167,184,235],"paired":[31],"answers":[32],"were":[33,72,80,196],"created":[34],"ophthalmologists":[36],"represent":[38],"commonly":[39],"asked":[40],"questions,":[42],"divided":[43],"into":[44],"fine-tuning":[45],"(368;":[46],"92%),":[47],"testing":[49,65,78],"(40;":[50],"8%).":[51],"We":[52],"find-tuned":[53],"5":[54,83,118],"different":[55],"LLMs,":[56,127],"including":[57],"LLAMA2-7b,":[58],"LLAMA2-7b-Chat,":[59],"LLAMA2-13b,":[60],"LLAMA2-13b-Chat.":[62],"For":[63],"dataset,":[66],"additional":[67],"8":[68],"glaucoma":[69,186],"QnA":[70],"pairs":[71],"included.":[73],"200":[74],"dataset":[79],"LLMs":[85],"evaluation.":[87,148,202],"A":[88],"customized":[89],"clinical":[90,101,121,189,206,217],"rubric":[92],"was":[93,112,176],"used":[94],"guide":[96],"GPT-4":[97,110,147,149,201,209],"evaluation,":[98],"grounded":[99],"on":[100,145,173],"accuracy,":[102],"relevance,":[103],"safety,":[105],"ease":[107],"understanding.":[109],"then":[113],"compared":[114],"against":[115],"ranking":[116],"clinicians":[119],"alignment.":[122],"Results:":[123],"Among":[124],"all":[125],"GPT-3.5":[128],"scored":[129],"highest":[131],"(87.1%),":[132],"followed":[133],"LLAMA2-13b":[135],"(80.9%),":[136],"LLAMA2-13b-chat":[137],"(75.5%),":[138],"LLAMA2-7b-Chat":[139],"(70%)":[140],"LLAMA2-7b":[142],"(68.8%)":[143],"based":[144,172],"demonstrated":[151],"significant":[152],"agreement":[153],"with":[154,158],"rankings,":[157],"Spearman":[159],"Kendall":[161],"Tau":[162],"correlation":[163,171],"coefficients":[164],"0.90":[166],"0.80":[168],"respectively;":[169],"while":[170],"Cohen":[174],"Kappa":[175],"more":[177],"modest":[178],"at":[179],"0.50.":[180],"Notably,":[181],"qualitative":[182],"analysis":[183],"sub-analysis":[187],"revealed":[188],"inaccuracies":[190],"in":[191,245],"LLM-generated":[193],"responses,":[194],"which":[195],"appropriately":[197],"identified":[198],"Conclusion:":[203],"The":[204],"notable":[205],"highlighted":[211],"its":[212],"potential":[213],"streamline":[215],"chatbot":[221],"healthcare-related":[224],"queries.":[225],"By":[226],"complementing":[227],"existing":[229],"clinician-dependent":[230],"manual":[231],"grading,":[232],"this":[233],"efficient":[234],"automated":[236],"could":[238],"assist":[239],"validation":[241],"future":[243],"developments":[244],"applications":[247],"healthcare.":[249]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-03-10T16:38:18.471706","created_date":"2025-10-10T00:00:00"}