{"id":"https://openalex.org/W7201840411","doi":"https://doi.org/10.2196/preprints.108828","title":"Characterizing large language model generative artificial intelligence variability in the production of objective structured clinical examination stations (Preprint)","display_name":"Characterizing large language model generative artificial intelligence variability in the production of objective structured clinical examination stations (Preprint)","publication_year":2026,"publication_date":"2026-08-06","ids":{"openalex":"https://openalex.org/W7201840411","doi":"https://doi.org/10.2196/preprints.108828"},"language":"en","primary_location":{"id":"doi:10.2196/preprints.108828","doi":"https://doi.org/10.2196/preprints.108828","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.108828","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100086938","display_name":"Kristen Joseph-Delaffon","orcid":"https://orcid.org/0009-0004-1071-1540","observed_orcids":["https://orcid.org/0009-0004-1071-1540"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kristen Joseph-Delaffon","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0009-0004-1071-1540","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082789684","display_name":"Maxime Desgrouas","orcid":"https://orcid.org/0000-0002-7713-2293","observed_orcids":["https://orcid.org/0000-0002-7713-2293"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maxime Desgrouas","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-7713-2293","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053553878","display_name":"Sophie Catanese","orcid":"https://orcid.org/0000-0001-6741-9402","observed_orcids":["https://orcid.org/0000-0001-6741-9402"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sophie Catanese","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-6741-9402","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037073564","display_name":"Julien Lejeune","orcid":"https://orcid.org/0000-0002-3862-9475","observed_orcids":["https://orcid.org/0000-0002-3862-9475"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Julien Lejeune","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-3862-9475","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5145925712","display_name":"Julien Nait-Kaci","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Julien Nait-Kaci","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022287678","display_name":"\u00c9ric Piver","orcid":"https://orcid.org/0000-0002-7101-0121","observed_orcids":["https://orcid.org/0000-0002-7101-0121"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Eric Piver","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-7101-0121","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116186665","display_name":"Isaure Breteau","orcid":"https://orcid.org/0000-0002-6399-188X","observed_orcids":["https://orcid.org/0000-0002-6399-188X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Isaure Breteau","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-6399-188X","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001564881","display_name":"Sophie Leducq","orcid":"https://orcid.org/0000-0002-4860-6365","observed_orcids":["https://orcid.org/0000-0002-4860-6365"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sophie Leducq","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-4860-6365","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078724215","display_name":"Philippe Gatault","orcid":"https://orcid.org/0000-0003-1664-2879","observed_orcids":["https://orcid.org/0000-0003-1664-2879"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Philippe Gatault","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-1664-2879","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061390687","display_name":"Raoul Kanav Khanna","orcid":"https://orcid.org/0000-0001-6032-1488","observed_orcids":["https://orcid.org/0000-0001-6032-1488"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raoul Kanav Khanna","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-6032-1488","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051306204","display_name":"Denis Angoulvant","orcid":"https://orcid.org/0000-0003-0788-8092","observed_orcids":["https://orcid.org/0000-0003-0788-8092"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Denis Angoulvant","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-0788-8092","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5059853582","display_name":"Nicolas Vallet","orcid":"https://orcid.org/0000-0001-9568-3358","observed_orcids":["https://orcid.org/0000-0001-9568-3358"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nicolas Vallet","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-9568-3358","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10254","display_name":"Innovations in Medical Education","score":0.026100000366568565,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14414","display_name":"Artificial Intelligence in Education","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.800000011920929},{"id":"https://openalex.org/keywords/generative-ai","display_name":"generative AI","score":0.9480000138282776},{"id":"https://openalex.org/keywords/objective-structured-clinical-examination","display_name":"OSCE","score":0.8450000286102295},{"id":"https://openalex.org/keywords/inter-model-variability","display_name":"inter-model variability","score":0.8349999785423279},{"id":"https://openalex.org/keywords/neonatal-jaundice","display_name":"neonatal jaundice","score":0.9919999837875366},{"id":"https://openalex.org/keywords/soft-skills-assessment","display_name":"soft skills assessment","score":0.6610000133514404},{"id":"https://openalex.org/keywords/content-validity","display_name":"content validity","score":0.6060000061988831},{"id":"https://openalex.org/keywords/chatgpt","display_name":"ChatGPT","score":0.7129999995231628},{"id":"https://openalex.org/keywords/gemini","display_name":"Gemini","score":0.9110000133514404},{"id":"https://openalex.org/keywords/claude","display_name":"Claude","score":0.828000009059906},{"id":"https://openalex.org/keywords/grok","display_name":"Grok","score":0.890999972820282}],"concepts":[{"id":"https://openalex.org/C2778000598","wikidata":"https://www.wikidata.org/wiki/Q11237593","display_name":"Objective structured clinical examination","level":2,"score":0.6403999924659729},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.6164000034332275},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.5835999846458435},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5619999766349792},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5080999732017517},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48649999499320984},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.35920000076293945},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3560999929904938},{"id":"https://openalex.org/C2780615836","wikidata":"https://www.wikidata.org/wiki/Q2471869","display_name":"USable","level":2,"score":0.353300005197525},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.3416999876499176},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3165000081062317},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.3025999963283539},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.29100000858306885},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.28679999709129333},{"id":"https://openalex.org/C146849305","wikidata":"https://www.wikidata.org/wiki/Q370766","display_name":"Ground truth","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.2624000012874603},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.25850000977516174},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.2531000077724457}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.2196/preprints.108828","doi":"https://doi.org/10.2196/preprints.108828","is_oa":false,"landing_page_url":"https://doi.org/10.2196/preprints.108828","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"posted-content"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Good health and well-being","score":0.8371999859809875,"id":"https://metadata.un.org/sdg/3"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W2164973508","https://openalex.org/W2165171258","https://openalex.org/W4400983317","https://openalex.org/W4407965332","https://openalex.org/W4409210035","https://openalex.org/W4414923740","https://openalex.org/W4416618182","https://openalex.org/W7122811845","https://openalex.org/W7124463484","https://openalex.org/W7124488972","https://openalex.org/W7162420219"],"related_works":["https://openalex.org/W7197004964","https://openalex.org/W7162420219","https://openalex.org/W7124488972","https://openalex.org/W7222986902","https://openalex.org/W7124463484","https://openalex.org/W7160541738","https://openalex.org/W7203975634","https://openalex.org/W7223004024","https://openalex.org/W7208825961","https://openalex.org/W4416618182"],"abstract_inverted_index":{"BACKGROUND":[0],"Designing":[1],"high-quality":[2],"Objective":[3],"Structured":[4],"Clinical":[5,212],"Examination":[6],"(OSCE)":[7],"stations":[8,58],"is":[9,37],"a":[10,18,283],"time-consuming":[11],"process.":[12],"Generative":[13],"artificial":[14],"intelligence":[15],"(AI)":[16],"represents":[17],"promising":[19],"path":[20],"to":[21,55,100],"accelerate":[22],"content":[23,271],"creation":[24],"by":[25],"automating":[26],"the":[27,46,60,72,90,95,177,277],"generation":[28],"of":[29,34,62,163,170,235,289],"scenarios.":[30],"A":[31,65],"growing":[32],"number":[33],"AI":[35,50,82],"tools":[36],"now":[38],"available":[39],"for":[40,77,111,127,154,279],"this":[41,87],"purpose.":[42],"OBJECTIVE":[43],"To":[44],"assess":[45],"variability":[47],"between":[48,148],"generative":[49],"models":[51,83,232,246],"in":[52,59,193,260,282],"their":[53],"ability":[54],"produce":[56],"OSCE":[57,75,104,251],"field":[61],"paediatrics.":[63],"METHODS":[64],"structured":[66],"prompt":[67],"was":[68,109,125,132,151,215],"developed":[69],"based":[70],"on":[71],"French":[73,96],"national":[74],"guidelines":[76],"medical":[78],"education.":[79],"Five":[80],"distinct":[81],"were":[84,191],"provided":[85],"with":[86,135,196],"prompt,":[88],"alongside":[89],"neonatal":[91,202],"jaundice":[92,203],"chapter":[93],"from":[94],"pediatric":[97],"reference":[98],"textbook,":[99],"generate":[101,248],"6":[102],"complete":[103],"stations.":[105],"RESULTS":[106],"Prompt":[107],"compliance":[108,255],"high":[110],"ChatGPT":[112,114,174,197],"5.1,":[113],"5.2,":[115],"Gemini":[116,181],"3.0":[117],"Pro,":[118],"and":[119,168,180,265,294],"Claude":[120],"Opus":[121],"4.5,":[122],"while":[123],"it":[124],"lower":[126],"Grok":[128],"4.1.":[129],"Expert-rated":[130],"quality":[131,156],"generally":[133],"high,":[134],"few":[136],"factual":[137],"errors":[138],"or":[139],"missing":[140],"information":[141],"across":[142,210,231],"models.":[143,149,211],"However":[144],"usability":[145],"differed":[146,224],"significantly":[147,230],"This":[150],"also":[152],"true":[153],"several":[155],"dimensions":[157],"such":[158],"as":[159,186],"checklist":[160,164],"clarity,":[161],"embedding":[162],"answers":[165],"within":[166],"vignettes,":[167],"ease":[169],"standardized":[171],"patient":[172],"formation.":[173],"5.1":[175,198],"required":[176],"most":[178,182],"revisions":[179],"often":[183],"rated":[184],"usable":[185],"is.":[187],"Significant":[188],"inter-model":[189,258],"differences":[190,259],"observed":[192],"diagnostics,":[194],"only":[195],"sampling":[199],"all":[200,239],"three":[201],"categories.":[204],"Contextual":[205],"variables":[206],"showed":[207],"systematic":[208],"narrowing":[209],"grid":[213],"density":[214],"consistent":[216],"(10-12":[217],"items":[218],"per":[219],"station),":[220],"but":[221,253],"thematic":[222],"distribution":[223],"markedly.":[225],"Soft":[226],"skills":[227,267],"coverage":[228],"varied":[229],"(p=0.002),":[233],"none":[234],"them":[236],"consistently":[237],"representing":[238],"communication":[240],"competency":[241],"domains.":[242],"CONCLUSIONS":[243],"Large":[244],"language":[245],"can":[247],"structurally":[249],"compliant":[250],"stations,":[252],"surface":[254],"conceals":[256],"substantive":[257],"diagnostic":[261],"coverage,":[262],"contextual":[263],"diversity,":[264],"soft":[266],"representation,":[268],"that":[269],"compromise":[270],"validity.":[272],"No":[273],"model":[274,290],"currently":[275],"meets":[276],"criteria":[278],"unsupervised":[280],"deployment":[281],"summative":[284],"assessment":[285,303],"bank.":[286],"The":[287],"choice":[288],"carries":[291],"pedagogical":[292],"implications":[293],"expert":[295],"curation":[296],"remains":[297],"essential":[298],"before":[299],"integration":[300],"into":[301],"high-stakes":[302],"workflows.":[304]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2026-08-08T00:00:00"}