{"id":"https://openalex.org/W7214023326","doi":"https://doi.org/10.1371/journal.pone.0358873","title":"Variability among large language models in assessing CONSORT compliance of published randomized clinical trials","display_name":"Variability among large language models in assessing CONSORT compliance of published randomized clinical trials","publication_year":2026,"publication_date":"2026-09-22","ids":{"openalex":"https://openalex.org/W7214023326","doi":"https://doi.org/10.1371/journal.pone.0358873","pmid":"https://pubmed.ncbi.nlm.nih.gov/42771600","pmcid":"https://www.ncbi.nlm.nih.gov/pmc/articles/13596804"},"language":"en","primary_location":{"id":"doi:10.1371/journal.pone.0358873","doi":"https://doi.org/10.1371/journal.pone.0358873","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0358873","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0358873&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0358873&type=printable","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119880528","display_name":"Daniel Y. Tsybulnik","orcid":"https://orcid.org/0009-0009-1710-1901","observed_orcids":["https://orcid.org/0009-0009-1710-1901"]},"institutions":[{"id":"https://openalex.org/I72951846","display_name":"Washington State University","ror":"https://ror.org/05dk0ce17","country_code":"US","type":"education","lineage":["https://openalex.org/I72951846"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Y. Tsybulnik","raw_affiliation_strings":["Elson S. Floyd College of Medicine, Washington State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Elson S. Floyd College of Medicine, Washington State University","institution_ids":["https://openalex.org/I72951846"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029890972","display_name":"Justin J. Gillette","orcid":"https://orcid.org/0009-0004-3352-561X","observed_orcids":["https://orcid.org/0009-0004-3352-561X"]},"institutions":[{"id":"https://openalex.org/I72951846","display_name":"Washington State University","ror":"https://ror.org/05dk0ce17","country_code":"US","type":"education","lineage":["https://openalex.org/I72951846"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Justin J. Gillette","raw_affiliation_strings":["Elson S. Floyd College of Medicine, Washington State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Elson S. Floyd College of Medicine, Washington State University","institution_ids":["https://openalex.org/I72951846"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069125829","display_name":"Thomas F Heston","orcid":"https://orcid.org/0000-0002-5655-2512","observed_orcids":["https://orcid.org/0000-0002-5655-2512"]},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I72951846","display_name":"Washington State University","ror":"https://ror.org/05dk0ce17","country_code":"US","type":"education","lineage":["https://openalex.org/I72951846"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Thomas F. Heston","raw_affiliation_strings":["Elson S. Floyd College of Medicine, Washington State University","University of Washington School of Medicine"],"raw_orcid":"https://orcid.org/0000-0002-5655-2512","affiliations":[{"raw_affiliation_string":"Elson S. Floyd College of Medicine, Washington State University","institution_ids":["https://openalex.org/I72951846"]},{"raw_affiliation_string":"University of Washington School of Medicine","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5069125829"],"corresponding_institution_ids":["https://openalex.org/I201448701","https://openalex.org/I72951846"],"apc_list":{"value":2382,"currency":"USD","value_usd":2382},"apc_paid":{"value":2382,"currency":"USD","value_usd":2382},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.66498175,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"21","issue":"9","first_page":"e0358873","last_page":"e0358873"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.8910999894142151,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.05739999935030937,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13607","display_name":"Academic Publishing and Open Access","score":0.013199999928474426,"subfield":{"id":"https://openalex.org/subfields/3309","display_name":"Library and Information Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9049999713897705},{"id":"https://openalex.org/keywords/randomized-controlled-trials","display_name":"randomized controlled trials","score":0.6769999861717224},{"id":"https://openalex.org/keywords/gpt-4o","display_name":"GPT-4o","score":0.8270000219345093},{"id":"https://openalex.org/keywords/gemini-2-5-flash","display_name":"Gemini 2.5 Flash","score":0.9269999861717224},{"id":"https://openalex.org/keywords/claude-sonnet-4-6","display_name":"Claude Sonnet 4.6","score":0.9739999771118164},{"id":"https://openalex.org/keywords/inter-model-agreement","display_name":"inter-model agreement","score":0.8059999942779541},{"id":"https://openalex.org/keywords/peer-review","display_name":"peer review","score":0.5189999938011169},{"id":"https://openalex.org/keywords/trial-quality-assessment","display_name":"trial quality assessment","score":0.5640000104904175}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.6503000259399414},{"id":"https://openalex.org/C168563851","wikidata":"https://www.wikidata.org/wiki/Q1436668","display_name":"Randomized controlled trial","level":2,"score":0.6316999793052673},{"id":"https://openalex.org/C49290038","wikidata":"https://www.wikidata.org/wiki/Q2994654","display_name":"Consolidated Standards of Reporting Trials","level":3,"score":0.5952000021934509},{"id":"https://openalex.org/C38721330","wikidata":"https://www.wikidata.org/wiki/Q80056","display_name":"Sonnet","level":3,"score":0.5304999947547913},{"id":"https://openalex.org/C535046627","wikidata":"https://www.wikidata.org/wiki/Q30612","display_name":"Clinical trial","level":2,"score":0.4528999924659729},{"id":"https://openalex.org/C95190672","wikidata":"https://www.wikidata.org/wiki/Q815382","display_name":"Meta-analysis","level":2,"score":0.42820000648498535},{"id":"https://openalex.org/C203092338","wikidata":"https://www.wikidata.org/wiki/Q1340863","display_name":"Clinical endpoint","level":3,"score":0.39070001244544983},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.3776000142097473},{"id":"https://openalex.org/C2780439572","wikidata":"https://www.wikidata.org/wiki/Q919364","display_name":"Publication bias","level":3,"score":0.3743000030517578},{"id":"https://openalex.org/C99476002","wikidata":"https://www.wikidata.org/wiki/Q42297","display_name":"Analysis of variance","level":2,"score":0.35920000076293945},{"id":"https://openalex.org/C102959455","wikidata":"https://www.wikidata.org/wiki/Q7313954","display_name":"Repeated measures design","level":2,"score":0.31119999289512634},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.3111000061035156},{"id":"https://openalex.org/C512399662","wikidata":"https://www.wikidata.org/wiki/Q3505712","display_name":"Family medicine","level":1,"score":0.3050000071525574},{"id":"https://openalex.org/C204243189","wikidata":"https://www.wikidata.org/wiki/Q1363085","display_name":"Randomization","level":3,"score":0.29910001158714294},{"id":"https://openalex.org/C197934379","wikidata":"https://www.wikidata.org/wiki/Q2047938","display_name":"Adverse effect","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C2781460075","wikidata":"https://www.wikidata.org/wiki/Q1399332","display_name":"Compliance (psychology)","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.28610000014305115},{"id":"https://openalex.org/C2778653282","wikidata":"https://www.wikidata.org/wiki/Q60752967","display_name":"Trial registration","level":3,"score":0.2831000089645386},{"id":"https://openalex.org/C202953159","wikidata":"https://www.wikidata.org/wiki/Q382695","display_name":"Intention-to-treat analysis","level":3,"score":0.27000001072883606},{"id":"https://openalex.org/C148482608","wikidata":"https://www.wikidata.org/wiki/Q17007018","display_name":"Clinical study design","level":3,"score":0.26409998536109924},{"id":"https://openalex.org/C119423318","wikidata":"https://www.wikidata.org/wiki/Q7628405","display_name":"Study heterogeneity","level":3,"score":0.2615000009536743},{"id":"https://openalex.org/C82789193","wikidata":"https://www.wikidata.org/wiki/Q2142611","display_name":"Relative risk","level":3,"score":0.25290000438690186},{"id":"https://openalex.org/C136206449","wikidata":"https://www.wikidata.org/wiki/Q3556841","display_name":"Reporting bias","level":3,"score":0.2515999972820282}],"mesh":[{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016032","descriptor_name":"Randomized Controlled Trials as Topic","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true},{"descriptor_ui":"D019983","descriptor_name":"Guideline Adherence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true}],"locations_count":4,"locations":[{"id":"doi:10.1371/journal.pone.0358873","doi":"https://doi.org/10.1371/journal.pone.0358873","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0358873","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0358873&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},{"id":"pmid:42771600","doi":null,"is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/42771600","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PloS one","raw_type":"Journal Article"},{"id":"pmh:oai:pubmedcentral.nih.gov:13596804","doi":null,"is_oa":true,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/42771600/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLoS One","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:31089b6f538e4293b82a6238ed856617","doi":null,"is_oa":false,"landing_page_url":"https://doaj.org/article/31089b6f538e4293b82a6238ed856617","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"PLoS ONE, Vol 21, Iss 9, p e0358873 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1371/journal.pone.0358873","doi":"https://doi.org/10.1371/journal.pone.0358873","is_oa":true,"landing_page_url":"https://doi.org/10.1371/journal.pone.0358873","pdf_url":"https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0358873&type=printable","source":{"id":"https://openalex.org/S202381698","display_name":"PLoS ONE","issn_l":"1932-6203","issn":["1932-6203"],"is_oa":true,"is_in_doaj":true,"is_core":true,"listed_in":["anvur-class-a","anvur-scientific","cwts-core","doaj","dongbi-c","doyens","erih-plus","jufo-1","ki-jl-1","medline","norway-1","poland-100","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310315706","host_organization_name":"Public Library of Science","host_organization_lineage":["https://openalex.org/P4310315706"],"host_organization_lineage_names":["Public Library of Science"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"PLOS One","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","score":0.4043000042438507,"display_name":"Good health and well-being"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214023326.pdf","grobid_xml":"https://content.openalex.org/works/W7214023326.grobid-xml"},"referenced_works_count":19,"referenced_works":["https://openalex.org/W2049321668","https://openalex.org/W2105898936","https://openalex.org/W2495547003","https://openalex.org/W2902356319","https://openalex.org/W4385968944","https://openalex.org/W4387241792","https://openalex.org/W4394846163","https://openalex.org/W4399694676","https://openalex.org/W4399757216","https://openalex.org/W4400734098","https://openalex.org/W4401484139","https://openalex.org/W4402215122","https://openalex.org/W4406408916","https://openalex.org/W4406887664","https://openalex.org/W4408070566","https://openalex.org/W4408981244","https://openalex.org/W4409417738","https://openalex.org/W4410721345","https://openalex.org/W4413780260"],"related_works":["https://openalex.org/W4414903511","https://openalex.org/W7084615457","https://openalex.org/W4416361398","https://openalex.org/W4413780260","https://openalex.org/W4414534865","https://openalex.org/W4411521355","https://openalex.org/W4416292386","https://openalex.org/W6954742507","https://openalex.org/W4402620497","https://openalex.org/W4408216127"],"abstract_inverted_index":{"BACKGROUND:":[0],"Peer":[1],"review":[2],"processes":[3],"may":[4],"inadequately":[5],"assess":[6,37],"compliance":[7,77,94,109,130,156,245,256],"with":[8,120,250,274],"established":[9],"reporting":[10],"guidelines":[11,41],"such":[12],"as":[13,158,168,184],"the":[14,87,102,280,288],"Consolidated":[15],"Standards":[16],"of":[17,104,117,165,243],"Reporting":[18],"Trials":[19],"(CONSORT)":[20],"criteria.":[21],"Large":[22],"language":[23],"models":[24,36,90,194,270],"(LLMs)":[25],"demonstrate":[26],"potential":[27],"for":[28,290],"systematic":[29],"manuscript":[30,296],"evaluation;":[31],"however,":[32],"how":[33],"consistently":[34],"different":[35],"adherence":[38],"to":[39],"CONSORT":[40,80,93,129,244],"in":[42,54,91,240,246],"published":[43,53,247],"clinical":[44],"trials":[45,52],"remains":[46],"unexplored.":[47],"METHODS:":[48],"Twenty":[49],"randomized":[50,248],"controlled":[51],"immunology":[55],"journals":[56],"between":[57,89,193,269],"2015":[58],"and":[59,71,101,145,176,216,262],"2016":[60],"were":[61,208],"identified":[62,163,181],"through":[63],"PubMed.":[64],"Three":[65],"LLMs":[66,237],"(ChatGPT-4o,":[67],"Gemini":[68,138,173,213,226,258],"2.5":[69,139,174,214,227,259],"Flash,":[70],"Claude":[72,146,177,219,230,263],"Sonnet":[73,147,178,220,231,264],"4.6)":[74],"independently":[75],"assessed":[76],"across":[78],"37":[79],"2010":[81],"subpoints.":[82],"The":[83],"primary":[84],"endpoint":[85],"was":[86,271],"difference":[88],"mean":[92],"score.":[95],"Secondary":[96],"endpoints":[97],"included":[98],"inter-model":[99,285],"agreement":[100,268],"proportion":[103],"articles":[105],"meeting":[106,169,185],"a":[107,154,159,251],"90%":[108,155],"threshold.":[110,283],"Statistical":[111],"analysis":[112,116],"employed":[113],"repeated":[114],"measures":[115],"variance":[118],"(ANOVA)":[119],"post-hoc":[121],"pairwise":[122,206,276],"comparisons":[123,207],"(\u03b1":[124],"=":[125,196,203,234],"0.05).":[126],"RESULTS:":[127],"Mean":[128],"rates":[131],"were:":[132],"ChatGPT-4o":[133,162,217,254],"80.8%":[134],"(95%":[135,142,150],"CI":[136,143,151],"75.8-85.8%),":[137],"Flash":[140,175,215,228,260],"64.4%":[141],"58.0-70.8%),":[144],"4.6":[148,179,265],"54.7%":[149],"48.0-61.3%).":[152],"Using":[153],"threshold":[157],"quality":[160],"benchmark,":[161],"25%":[164],"papers":[166],"(5/20)":[167],"this":[170,186],"standard,":[171],"while":[172],"each":[180],"none":[182],"(0/20)":[183],"standard.":[187],"Repeated-measures":[188],"ANOVA":[189],"demonstrated":[190],"significant":[191,210],"differences":[192],"(F(2,38)":[195],"43.01,":[197],"p":[198,223,233],"<":[199,224],"0.001,":[200],"partial":[201],"\u03b7\u00b2":[202],"0.694).":[204],"All":[205],"statistically":[209],"(ChatGPT-4o":[211],"versus":[212,218,229],"4.6,":[221,232],"both":[222],"0.001;":[225],"0.014).":[235],"CONCLUSIONS:":[236],"varied":[238],"substantially":[239],"their":[241],"assessment":[242],"trials,":[249],"consistent":[252],"ordering:":[253],"scored":[255],"highest,":[257],"intermediate,":[261],"lowest.":[266],"Item-level":[267],"only":[272],"moderate,":[273],"no":[275],"weighted":[277],"kappa":[278],"reaching":[279],"0.61":[281],"substantial-agreement":[282],"This":[284],"variability":[286],"indicates":[287],"need":[289],"standardized":[291],"evaluation":[292],"protocols":[293],"before":[294],"LLM-assisted":[295],"screening":[297],"is":[298],"adopted.":[299]},"counts_by_year":[],"updated_date":"2026-10-31T12:09:11.096373","created_date":"2026-09-23T00:00:00"}