{"id":"https://openalex.org/W7214912866","doi":"https://doi.org/10.7759/cureus.117189","title":"Multimodal Large Language Models Rate Wetlab Suturing Repeatably but Fail to Detect Training-Related Improvement: A Pilot Study With Expert Assessment","display_name":"Multimodal Large Language Models Rate Wetlab Suturing Repeatably but Fail to Detect Training-Related Improvement: A Pilot Study With Expert Assessment","publication_year":2026,"publication_date":"2026-09-30","ids":{"openalex":"https://openalex.org/W7214912866","doi":"https://doi.org/10.7759/cureus.117189","pmid":"https://pubmed.ncbi.nlm.nih.gov/42964635","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/13720391"},"language":"en","primary_location":{"id":"doi:10.7759/cureus.117189","doi":"https://doi.org/10.7759/cureus.117189","is_oa":true,"landing_page_url":"https://doi.org/10.7759/cureus.117189","pdf_url":"https://www.cureus.com/articles/533738-multimodal-large-language-models-rate-wetlab-suturing-repeatably-but-fail-to-detect-training-related-improvement-a-pilot-study-with-expert-assessment.pdf","source":{"id":"https://openalex.org/S2738950867","display_name":"Cureus","issn_l":"2168-8184","issn":["2168-8184"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310316779","host_organization_name":"Cureus, Inc.","host_organization_lineage":["https://openalex.org/P4310316779"],"host_organization_lineage_names":["Cureus, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Cureus","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://www.cureus.com/articles/533738-multimodal-large-language-models-rate-wetlab-suturing-repeatably-but-fail-to-detect-training-related-improvement-a-pilot-study-with-expert-assessment.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055908733","display_name":"Jameel Mushtaq","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1325675289","display_name":"Moorfields Eye Hospital NHS Foundation Trust","ror":"https://ror.org/03zaddr67","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I1325675289"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Jameel Mushtaq","raw_affiliation_strings":["Ophthalmology, Moorfields Eye Hospital NHS Foundation Trust, London, GBR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ophthalmology, Moorfields Eye Hospital NHS Foundation Trust, London, GBR","institution_ids":["https://openalex.org/I1325675289"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062194534","display_name":"Zaid Alsafi","orcid":"https://orcid.org/0000-0002-1661-4782","observed_orcids":["https://orcid.org/0000-0002-1661-4782"]},"institutions":[{"id":"https://openalex.org/I153355300","display_name":"Imperial College Healthcare NHS Trust","ror":"https://ror.org/056ffv270","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I153355300"]},{"id":"https://openalex.org/I4210136817","display_name":"Western Eye Hospital","ror":"https://ror.org/04g0t2d47","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I153355300","https://openalex.org/I4210136817"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zaid Alsafi","raw_affiliation_strings":["Ophthalmology, Western Eye Hospital - Imperial College Healthcare NHS Trust, London, GBR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ophthalmology, Western Eye Hospital - Imperial College Healthcare NHS Trust, London, GBR","institution_ids":["https://openalex.org/I153355300","https://openalex.org/I4210136817"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5153594858","display_name":"Robert Brady","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1325675289","display_name":"Moorfields Eye Hospital NHS Foundation Trust","ror":"https://ror.org/03zaddr67","country_code":"GB","type":"healthcare","lineage":["https://openalex.org/I1325675289"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Robert Brady","raw_affiliation_strings":["Ophthalmology, Moorfields Eye Hospital NHS Foundation Trust, London, GBR"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ophthalmology, Moorfields Eye Hospital NHS Foundation Trust, London, GBR","institution_ids":["https://openalex.org/I1325675289"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5055908733"],"corresponding_institution_ids":["https://openalex.org/I1325675289"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.74555501,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"18","issue":"9","first_page":"e117189","last_page":"e117189"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T10916","display_name":"Surgical Simulation and Training","score":0.9804999828338623,"subfield":{"id":"https://openalex.org/subfields/2746","display_name":"Surgery"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.00860000029206276,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.005400000140070915,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multimodal-large-language-models","display_name":"multimodal large language models","score":0.9819999933242798},{"id":"https://openalex.org/keywords/suturing","display_name":"suturing","score":0.7269999980926514},{"id":"https://openalex.org/keywords/ophthalmic-surgery-training","display_name":"ophthalmic surgery training","score":0.7239999771118164},{"id":"https://openalex.org/keywords/repeatability","display_name":"repeatability","score":0.7139999866485596},{"id":"https://openalex.org/keywords/oculoplastic-surgery","display_name":"oculoplastic surgery","score":0.6610000133514404},{"id":"https://openalex.org/keywords/surgical-training","display_name":"surgical training","score":0.3869999945163727},{"id":"https://openalex.org/keywords/gpt-5-5","display_name":"GPT-5.5","score":0.9240000247955322},{"id":"https://openalex.org/keywords/porcine-tissue","display_name":"porcine tissue","score":0.5540000200271606},{"id":"https://openalex.org/keywords/ophthalmology-residents","display_name":"ophthalmology residents","score":0.4970000088214874}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.8622999787330627},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38119998574256897},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.3050000071525574},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.29490000009536743},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.28130000829696655},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.2476000040769577},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.24549999833106995},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.24539999663829803},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.23229999840259552},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.21480000019073486}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.7759/cureus.117189","doi":"https://doi.org/10.7759/cureus.117189","is_oa":true,"landing_page_url":"https://doi.org/10.7759/cureus.117189","pdf_url":"https://www.cureus.com/articles/533738-multimodal-large-language-models-rate-wetlab-suturing-repeatably-but-fail-to-detect-training-related-improvement-a-pilot-study-with-expert-assessment.pdf","source":{"id":"https://openalex.org/S2738950867","display_name":"Cureus","issn_l":"2168-8184","issn":["2168-8184"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310316779","host_organization_name":"Cureus, Inc.","host_organization_lineage":["https://openalex.org/P4310316779"],"host_organization_lineage_names":["Cureus, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Cureus","raw_type":"journal-article"},{"id":"pmh:oai:pubmedcentral.nih.gov:13720391","doi":null,"is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13720391/","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13720391/pdf/cureus-0018-00000117189.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Cureus","raw_type":"Text"},{"id":"pmid:42964635","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/42964635","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Cureus","raw_type":"Journal Article"}],"best_oa_location":{"id":"doi:10.7759/cureus.117189","doi":"https://doi.org/10.7759/cureus.117189","is_oa":true,"landing_page_url":"https://doi.org/10.7759/cureus.117189","pdf_url":"https://www.cureus.com/articles/533738-multimodal-large-language-models-rate-wetlab-suturing-repeatably-but-fail-to-detect-training-related-improvement-a-pilot-study-with-expert-assessment.pdf","source":{"id":"https://openalex.org/S2738950867","display_name":"Cureus","issn_l":"2168-8184","issn":["2168-8184"],"is_oa":true,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","poland-20","russia-white-list-2","vabb-shw"],"host_organization":"https://openalex.org/P4310316779","host_organization_name":"Cureus, Inc.","host_organization_lineage":["https://openalex.org/P4310316779"],"host_organization_lineage_names":["Cureus, Inc."],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Cureus","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","score":0.5683000087738037,"display_name":"Good health and well-being"},{"id":"https://metadata.un.org/sdg/4","score":0.4156000018119812,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214912866.pdf","grobid_xml":"https://content.openalex.org/works/W7214912866.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2109498513","https://openalex.org/W2164777277","https://openalex.org/W2327037637","https://openalex.org/W2510605706","https://openalex.org/W2561811553","https://openalex.org/W2951058133","https://openalex.org/W3038782798","https://openalex.org/W3127144036","https://openalex.org/W3134650884","https://openalex.org/W3213819867","https://openalex.org/W4368340908","https://openalex.org/W4392715916","https://openalex.org/W4394760737","https://openalex.org/W4396692989","https://openalex.org/W4400778524","https://openalex.org/W4412493746","https://openalex.org/W4413478063","https://openalex.org/W7117702663","https://openalex.org/W7167836367"],"related_works":["https://openalex.org/W7204203946","https://openalex.org/W4297474739","https://openalex.org/W7212185057","https://openalex.org/W7201831485","https://openalex.org/W7204872969","https://openalex.org/W4294993279","https://openalex.org/W4386273553","https://openalex.org/W4283258651","https://openalex.org/W7161024458","https://openalex.org/W4416880095"],"abstract_inverted_index":{"Background":[0],"Objective":[1],"assessment":[2,473],"of":[3,211,257,304,308,327,404,439,474],"technical":[4,66,476],"skill":[5],"remains":[6],"a":[7,53,69,86,93,105,119,239,284,325],"bottleneck":[8],"in":[9,99,238,265,330,392],"surgical":[10,65,475],"training.":[11],"Expert":[12],"rating":[13,413],"is":[14,22],"valid":[15],"but":[16,60,365,429],"consumes":[17],"scarce":[18],"faculty":[19],"time":[20],"and":[21,77,113,132,145,167,185,199,221,255,292,312,335,401,434,437,450,478,500],"susceptible":[23],"to":[24,40,63,83,134,142,146,160,175,235,273,306,314,347,363,373,379,442],"systematic":[25,435],"rater":[26],"effects,":[27],"while":[28],"existing":[29],"automated":[30],"methods":[31,78,479],"rely":[32],"on":[33,154],"bespoke":[34],"computer-vision":[35],"models":[36,47,180,337,422,466],"that":[37,446,480],"are":[38,467,486,503],"inaccessible":[39],"most":[41],"educators.":[42],"General-purpose":[43],"multimodal":[44,177,419],"large":[45,178,420],"language":[46,179,421],"can":[48,491],"score":[49,326],"an":[50,155,444,448],"image":[51,153,223],"against":[52],"written":[54],"rubric":[55,158,171,463],"with":[56,104,247,317,351,384,430],"no":[57,322],"task-specific":[58],"training,":[59],"their":[61,482],"ability":[62],"judge":[64],"quality":[67,357],"from":[68,203,497],"still":[70],"photograph":[71],"has":[72],"not":[73,456,468],"been":[74],"established.":[75],"Materials":[76],"Five":[79],"ophthalmology":[80],"residents":[81],"(ST3":[82],"ST6)":[84],"attending":[85],"face-to-face":[87],"oculoplastic":[88,139],"wetlab":[89,424],"course":[90],"each":[91,191],"closed":[92],"standardised":[94,130],"10":[95,124,206,501],"cm":[96],"linear":[97],"wound":[98],"ex":[100],"vivo":[101],"porcine":[102],"tissue":[103],"running":[106],"continuous":[107],"5-0":[108],"polypropylene":[109],"suture,":[110],"once":[111,114],"before":[112,488],"after":[115],"the":[116,135,168,204,214,222,233,248,301,320,331,341,352],"course,":[117],"within":[118],"fixed":[120],"10-minute":[121],"limit.":[122],"The":[123,164,208,229,261],"resulting":[125],"closures":[126],"were":[127,173,245],"photographed":[128],"under":[129],"conditions":[131],"cropped":[133],"wound.":[136],"A":[137],"senior":[138],"fellow,":[140],"blinded":[141],"delegate":[143,215],"identity":[144],"pre-":[147,234],"versus":[148],"post-course":[149,236],"status,":[150],"scored":[151],"every":[152,412],"eight-item":[156],"anchored":[157],"(1":[159],"5":[161,328],"per":[162],"item).":[163],"same":[165,169,205],"images":[166],"locked":[170],"prompt":[172],"submitted":[174],"three":[176,189,389,440],"(Claude":[181,398],"Opus":[182,289,298,399],"4.5,":[183],"GPT-5.5,":[184],"Gemini":[186,280],"3.1":[187,281,395],"Flash-Lite),":[188,396],"times":[190],"at":[192],"temperature":[193],"0,":[194],"yielding":[195],"90":[196],"model":[197,258,323,452],"ratings":[198,311],"720":[200,332],"item":[201,310,333,344],"scores":[202,386],"photographs.":[207],"independent":[209],"unit":[210],"analysis":[212],"was":[213,232,358,368,382],"(n":[216,224],"=":[217,225],"5)":[218],"for":[219,227,319,354,410,459,471],"change":[220,237,277],"10)":[226],"agreement.":[228],"primary":[230],"outcome":[231],"seven-item":[240],"composite":[241],"score;":[242],"secondary":[243],"outcomes":[244],"agreement":[246,350,367,371],"expert,":[249],"repeatability":[250],"across":[251,387],"identical":[252,385],"repeat":[253,390],"calls,":[254],"completeness":[256],"output.":[259],"Results":[260],"expert":[262,342,353,449],"recorded":[263],"improvement":[264,445],"all":[266,309,336,388],"five":[267,498],"delegates":[268,499],"(composite":[269],"2.90":[270],"\u00b1":[271,275],"0.65":[272,362],"3.71":[274],"0.49;":[276],"+0.82).":[278],"Only":[279],"Flash-Lite":[282],"reproduced":[283],"comparable":[285],"effect":[286],"(+0.77);":[287],"Claude":[288,297],"4.5":[290,299],"(+0.21)":[291],"GPT-5.5":[293,313],"(+0.19)":[294],"did":[295],"not.":[296],"assigned":[300],"scale":[302],"midpoint":[303],"3":[305],"72.3%":[307],"68.0%,":[315],"compared":[316],"39.2%":[318],"expert;":[321],"awarded":[324],"anywhere":[329],"scores,":[334],"rated":[338,423],"systematically":[339],"below":[340],"(mean":[343],"difference":[345],"-0.45":[346],"-0.73).":[348],"Rank":[349],"overall":[355],"closure":[356],"moderate":[359],"(Spearman":[360],"rho":[361],"0.79),":[364],"item-level":[366],"poor":[369],"(exact":[370],"33%":[372],"42%;":[374],"quadratic":[375],"weighted":[376],"kappa":[377],"0.20":[378],"0.24).":[380],"Repeatability":[381,454],"high,":[383],"calls":[391],"100%":[393],"(Gemini":[394],"91%":[397],"4.5),":[400],"70%":[402],"(GPT-5.5)":[403],"image-item":[405],"pairs.":[406],"Knot":[407],"security":[408],"accounted":[409],"almost":[411],"returned":[414],"as":[415],"non-assessable.":[416],"Conclusions":[417],"Contemporary":[418],"suturing":[425],"photographs":[426],"highly":[427],"repeatably":[428],"severe":[431],"central-tendency":[432],"compression":[433],"severity,":[436],"two":[438],"failed":[441],"detect":[443],"both":[447],"one":[451],"identified.":[453],"should":[455],"be":[457,492],"mistaken":[458],"validity.":[460],"With":[461],"single-image":[462],"prompting,":[464],"these":[465],"yet":[469],"suitable":[470],"summative":[472],"skill,":[477],"widen":[481],"effective":[483],"scoring":[484],"range":[485],"required":[487],"formative":[489],"use":[490],"recommended.":[493],"These":[494],"pilot":[495],"findings,":[496],"images,":[502],"hypothesis-generating.":[504]},"counts_by_year":[],"updated_date":"2026-11-08T12:12:28.789561","created_date":"2026-10-01T00:00:00"}