{"id":"https://openalex.org/W7140102396","doi":"https://doi.org/10.48550/arxiv.2603.19303","title":"Agreement Between Large Language Models, Human Reviewers, and Authors in Evaluating STROBE Checklists for Observational Studies in Rheumatology","display_name":"Agreement Between Large Language Models, Human Reviewers, and Authors in Evaluating STROBE Checklists for Observational Studies in Rheumatology","publication_year":2026,"publication_date":"2026-03-12","ids":{"openalex":"https://openalex.org/W7140102396","doi":"https://doi.org/10.48550/arxiv.2603.19303"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.19303","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.19303","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.19303","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5127126006","display_name":"EMRE B\u0130LG\u0130N","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Bilgin, Emre","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101718650","display_name":"Ebru \u00d6zt\u00fcrk","orcid":"https://orcid.org/0000-0003-1843-3499"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ozturk, Ebru","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130395544","display_name":"Meera Shah","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shah, Meera","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130405846","display_name":"Lisa Traboco","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Traboco, Lisa","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130396511","display_name":"Rebecca Everitt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Everitt, Rebecca","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130378209","display_name":"Ai Lyn Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Ai Lyn","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039496972","display_name":"Marwan Bukhari","orcid":"https://orcid.org/0000-0003-4311-5222"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bukhari, Marwan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130392613","display_name":"Vincenzo Venerito","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Venerito, Vincenzo","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5130355569","display_name":"Latika Gupta","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gupta, Latika","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5127126006"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.39259999990463257,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13309","display_name":"Reliability and Agreement in Measurement","score":0.39259999990463257,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10206","display_name":"Meta-analysis and systematic reviews","score":0.1370999962091446,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12443","display_name":"Delphi Technique in Research","score":0.06679999828338623,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/observational-study","display_name":"Observational study","score":0.8341000080108643},{"id":"https://openalex.org/keywords/strengthening-the-reporting-of-observational-studies-in-epidemiology","display_name":"Strengthening the reporting of observational studies in epidemiology","score":0.6541000008583069},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6348000168800354},{"id":"https://openalex.org/keywords/presentation","display_name":"Presentation (obstetrics)","score":0.3961000144481659},{"id":"https://openalex.org/keywords/checklist","display_name":"Checklist","score":0.3871999979019165},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.3711000084877014},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.3231000006198883}],"concepts":[{"id":"https://openalex.org/C23131810","wikidata":"https://www.wikidata.org/wiki/Q818574","display_name":"Observational study","level":2,"score":0.8341000080108643},{"id":"https://openalex.org/C2779638118","wikidata":"https://www.wikidata.org/wiki/Q7623265","display_name":"Strengthening the reporting of observational studies in epidemiology","level":3,"score":0.6541000008583069},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6348000168800354},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.45899999141693115},{"id":"https://openalex.org/C2777601897","wikidata":"https://www.wikidata.org/wiki/Q3409113","display_name":"Presentation (obstetrics)","level":2,"score":0.3961000144481659},{"id":"https://openalex.org/C2779356329","wikidata":"https://www.wikidata.org/wiki/Q922625","display_name":"Checklist","level":2,"score":0.3871999979019165},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.3711000084877014},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.3594000041484833},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.351500004529953},{"id":"https://openalex.org/C509550671","wikidata":"https://www.wikidata.org/wiki/Q126945","display_name":"Medical education","level":1,"score":0.335099995136261},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C61863361","wikidata":"https://www.wikidata.org/wiki/Q470749","display_name":"Inter-rater reliability","level":3,"score":0.3188000023365021},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.30000001192092896},{"id":"https://openalex.org/C71008984","wikidata":"https://www.wikidata.org/wiki/Q2890076","display_name":"Rigour","level":2,"score":0.2809000015258789},{"id":"https://openalex.org/C2778653282","wikidata":"https://www.wikidata.org/wiki/Q60752967","display_name":"Trial registration","level":3,"score":0.27810001373291016},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.2768999934196472},{"id":"https://openalex.org/C512399662","wikidata":"https://www.wikidata.org/wiki/Q3505712","display_name":"Family medicine","level":1,"score":0.27559998631477356},{"id":"https://openalex.org/C2777083192","wikidata":"https://www.wikidata.org/wiki/Q1814648","display_name":"Plain language","level":2,"score":0.27090001106262207},{"id":"https://openalex.org/C40993552","wikidata":"https://www.wikidata.org/wiki/Q514654","display_name":"Gold standard (test)","level":2,"score":0.26840001344680786},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.26649999618530273},{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.25110000371932983}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.19303","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.19303","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.19303","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.19303","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being","score":0.7201820015907288}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Introduction:":[0],"Evaluating":[1],"compliance":[2],"with":[3,137,147,180,192,213],"the":[4,6,35,46,61,67,159,165,172,178,181],"Strengthening":[5],"Reporting":[7],"of":[8],"Observational":[9],"Studies":[10],"in":[11,39,243],"Epidemiology":[12],"(STROBE)":[13],"statement":[14],"can":[15],"be":[16],"time-consuming":[17],"and":[18,34,48,79,90,92,121,124,150,171],"subjective.":[19],"This":[20],"study":[21],"compares":[22],"STROBE":[23,63,208],"assessments":[24],"from":[25,74],"large":[26],"language":[27],"models":[28,229],"(LLMs),":[29],"a":[30,69,222],"human":[31,71,139,148,193,214,241],"reviewer":[32,174],"panel,":[33],"original":[36],"manuscript":[37],"authors":[38,151,182],"observational":[40,245],"rheumatology":[41,54],"research.":[42,246],"Methods:":[43],"Guided":[44],"by":[45,66],"GRRAS":[47],"DEAL":[49],"Pathway":[50],"B":[51],"frameworks,":[52],"17":[53],"articles":[55],"were":[56,85],"independently":[57],"assessed.":[58],"Evaluations":[59],"used":[60],"22-item":[62],"checklist,":[64],"completed":[65],"authors,":[68],"five-person":[70],"panel":[72],"(ranging":[73],"junior":[75],"to":[76,163],"senior":[77,173],"professionals),":[78],"two":[80],"LLMs":[81,132,203],"(ChatGPT-5.2,":[82],"Gemini-3Pro).":[83],"Items":[84],"grouped":[86],"into":[87],"Methodological":[88,128],"Rigor":[89,129],"Presentation":[91,120],"Context":[93,122],"domains.":[94],"Inter-rater":[95],"reliability":[96],"was":[97,110,175,183],"calculated":[98],"using":[99],"Gwet's":[100],"Agreement":[101],"Coefficient":[102],"(AC1).":[103],"Results:":[104],"Overall":[105],"agreement":[106,118,126,135,146,166,179,191,212],"across":[107],"all":[108,138],"reviewers":[109,140,149,194],"85.0%":[111],"(AC1=0.826).":[112],"Domain":[113],"stratification":[114],"showed":[115],"almost":[116],"perfect":[117],"for":[119,127,206,233,238],"(AC1=0.841)":[123],"substantial":[125],"(AC1=0.803).":[130],"Although":[131],"achieved":[133],"complete":[134],"(AC1=1.000)":[136],"on":[141,153,161,197,216,224],"standard":[142],"formatting":[143],"elements,":[144],"their":[145,210],"declined":[152],"complex":[154,217],"items.":[155,200],"For":[156],"example,":[157],"regarding":[158],"item":[160],"loss":[162],"follow-up,":[164],"between":[167],"Gemini":[168],"3":[169],"Pro":[170],"AC1=-0.252,":[176],"while":[177],"only":[184],"fair.":[185],"Additionally,":[186],"ChatGPT-5.2":[187],"generally":[188],"demonstrated":[189],"higher":[190],"than":[195,237],"Gemini-3Pro":[196],"specific":[198],"methodological":[199,218],"Conclusion:":[201],"While":[202],"show":[204],"potential":[205],"basic":[207],"screening,":[209],"lower":[211],"experts":[215],"items":[219],"likely":[220],"reflects":[221],"reliance":[223],"surface-level":[225],"information.":[226],"Currently,":[227],"these":[228],"appear":[230],"more":[231],"reliable":[232],"standardizing":[234],"straightforward":[235],"checks":[236],"replacing":[239],"expert":[240],"judgment":[242],"evaluating":[244]},"counts_by_year":[],"updated_date":"2026-03-24T06:04:31.470712","created_date":"2026-03-24T00:00:00"}