{"id":"https://openalex.org/W7224076846","doi":"https://doi.org/10.48550/arxiv.2610.40318","title":"Decision and System One models in rheumatology: accuracy and calibration on specialty-training examination questions and diagnostic case vignettes","display_name":"Decision and System One models in rheumatology: accuracy and calibration on specialty-training examination questions and diagnostic case vignettes","publication_year":2026,"publication_date":"2026-10-28","ids":{"openalex":"https://openalex.org/W7224076846","doi":"https://doi.org/10.48550/arxiv.2610.40318"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2610.40318","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40318","pdf_url":"https://arxiv.org/pdf/2610.40318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2610.40318","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077234403","display_name":"Alfredo Madrid-Garc\u00eda","orcid":"https://orcid.org/0000-0002-1591-0467","observed_orcids":["https://orcid.org/0000-0002-1591-0467"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Madrid-Garc\u00eda, Alfredo","raw_affiliation_strings":["Independent researcher , Madrid , Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent researcher , Madrid , Spain","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069000959","display_name":"Diego Benavent","orcid":"https://orcid.org/0000-0001-9119-5330","observed_orcids":["https://orcid.org/0000-0001-9119-5330"]},"institutions":[{"id":"https://openalex.org/I2802050225","display_name":"Bellvitge University Hospital","ror":"https://ror.org/00epner96","country_code":"ES","type":"healthcare","lineage":["https://openalex.org/I157913058","https://openalex.org/I2802050225","https://openalex.org/I4210153292"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Benavent, Diego","raw_affiliation_strings":["Rheumatology Department , Hospital Universitari de Bellvitge , Barcelona , Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Rheumatology Department , Hospital Universitari de Bellvitge , Barcelona , Spain","institution_ids":["https://openalex.org/I2802050225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5092817899","display_name":"Fabian Lechner","orcid":"https://orcid.org/0009-0003-9200-477X","observed_orcids":["https://orcid.org/0009-0003-9200-477X"]},"institutions":[{"id":"https://openalex.org/I161103922","display_name":"Philipps University of Marburg","ror":"https://ror.org/01rdrb571","country_code":"DE","type":"education","lineage":["https://openalex.org/I161103922"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Lechner, Fabian","raw_affiliation_strings":["Institute for Digital Medicine , Marburg University , Marburg , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Digital Medicine , Marburg University , Marburg , Germany","institution_ids":["https://openalex.org/I161103922"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085779260","display_name":"Phillip Kremer","orcid":"https://orcid.org/0009-0009-9897-2302","observed_orcids":["https://orcid.org/0009-0009-9897-2302"]},"institutions":[{"id":"https://openalex.org/I4210108711","display_name":"University Medical Center Hamburg-Eppendorf","ror":"https://ror.org/01zgy1s35","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I159176309","https://openalex.org/I4210108711"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Kremer, Phillip","raw_affiliation_strings":["Division of Rheumatology and Systemic Inflammatory Disorders , University Medical Center Hamburg-Eppendorf , Hamburg , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Division of Rheumatology and Systemic Inflammatory Disorders , University Medical Center Hamburg-Eppendorf , Hamburg , Germany","institution_ids":["https://openalex.org/I4210108711"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081745118","display_name":"Johannes Knitza","orcid":"https://orcid.org/0000-0001-9695-0657","observed_orcids":["https://orcid.org/0000-0001-9695-0657","https://orcid.org/0009-0005-9192-4026"]},"institutions":[{"id":"https://openalex.org/I161103922","display_name":"Philipps University of Marburg","ror":"https://ror.org/01rdrb571","country_code":"DE","type":"education","lineage":["https://openalex.org/I161103922"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Knitza, Johannes","raw_affiliation_strings":["Alfredo Madrid-Garc\u00eda , Madrid , Spain","Institute for Digital Medicine , Marburg University , Marburg , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alfredo Madrid-Garc\u00eda , Madrid , Spain","institution_ids":[]},{"raw_affiliation_string":"Institute for Digital Medicine , Marburg University , Marburg , Germany","institution_ids":["https://openalex.org/I161103922"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059081477","display_name":"Beatriz Merino Barbancho","orcid":"https://orcid.org/0000-0001-5070-4178","observed_orcids":["https://orcid.org/0000-0001-5070-4178"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Merino-Barbancho, Beatriz","raw_affiliation_strings":["Independent researcher , Madrid , Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Independent researcher , Madrid , Spain","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.001500000013038516,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13946","display_name":"Musculoskeletal Disorders and Rehabilitation","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/2742","display_name":"Rehabilitation"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rheumatology","display_name":"rheumatology","score":0.6600000262260437},{"id":"https://openalex.org/keywords/decision-models","display_name":"decision models","score":0.4650000035762787},{"id":"https://openalex.org/keywords/jev","display_name":"Jev","score":0.6200000047683716},{"id":"https://openalex.org/keywords/diagnostic-accuracy","display_name":"diagnostic accuracy","score":0.4909999966621399},{"id":"https://openalex.org/keywords/calibration","display_name":"calibration","score":0.7129999995231628},{"id":"https://openalex.org/keywords/overconfidence","display_name":"overconfidence","score":0.6679999828338623},{"id":"https://openalex.org/keywords/linguistic-differences","display_name":"language differences","score":0.6159999966621399},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.5860000252723694},{"id":"https://openalex.org/keywords/clinical-decision-support","display_name":"clinical decision support","score":0.49300000071525574},{"id":"https://openalex.org/keywords/medical-education-evaluation","display_name":"medical education assessment","score":0.3230000138282776},{"id":"https://openalex.org/keywords/generative-ai-in-medicine","display_name":"generative AI in medicine","score":0.46299999952316284}],"concepts":[{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.6486999988555908},{"id":"https://openalex.org/C107763842","wikidata":"https://www.wikidata.org/wiki/Q181040","display_name":"Clef","level":3,"score":0.5842999815940857},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5134000182151794},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5097000002861023},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.490200012922287},{"id":"https://openalex.org/C2777382242","wikidata":"https://www.wikidata.org/wiki/Q6017816","display_name":"Index (typography)","level":2,"score":0.46459999680519104},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.40529999136924744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3880999982357025},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.3330000042915344},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3319000005722046},{"id":"https://openalex.org/C2780033181","wikidata":"https://www.wikidata.org/wiki/Q24962805","display_name":"Decision aids","level":3,"score":0.33000001311302185},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.31450000405311584},{"id":"https://openalex.org/C59594135","wikidata":"https://www.wikidata.org/wiki/Q5249242","display_name":"Decision model","level":2,"score":0.2919999957084656},{"id":"https://openalex.org/C3020132585","wikidata":"https://www.wikidata.org/wiki/Q2671652","display_name":"Diagnostic accuracy","level":2,"score":0.2849000096321106},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.28119999170303345}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2610.40318","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40318","pdf_url":"https://arxiv.org/pdf/2610.40318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2610.40318","doi":"https://doi.org/10.48550/arxiv.2610.40318","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2610.40318","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23741215","doi":"https://doi.org/10.5281/zenodo.23741215","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23741215","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23741216","doi":"https://doi.org/10.5281/zenodo.23741216","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23741216","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2610.40318","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2610.40318","pdf_url":"https://arxiv.org/pdf/2610.40318","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Good health and well-being","score":0.504800021648407,"id":"https://metadata.un.org/sdg/3"}],"study_designs":[],"awards":[{"id":"https://openalex.org/G3325143442","display_name":"Stratification of Patients using advanced Integrative modeling of Data Routinely acquired for diagnosing Rheumatic complaints","funder_award_id":"101080711","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7490757072","display_name":null,"funder_award_id":"101080711","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320307765","display_name":"Pfizer","ror":"https://ror.org/01xdqrp08"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320327470","display_name":"Swedish Orphan Biovitrum","ror":"https://ror.org/04asvbz37"},{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7224076846.pdf","grobid_xml":"https://content.openalex.org/works/W7224076846.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7158616730","https://openalex.org/W7204884325","https://openalex.org/W7134288457","https://openalex.org/W7119716229","https://openalex.org/W4389685420","https://openalex.org/W4402949381","https://openalex.org/W4417153090","https://openalex.org/W4403312492","https://openalex.org/W4416872242","https://openalex.org/W4386998832"],"abstract_inverted_index":{"Objective:":[0],"To":[1],"evaluate":[2],"the":[3,56,79,106,114,118,315],"accuracy":[4,126,141,195,234,284],"and":[5,9,17,38,40,70,92,146,168,170,173,186,211,240,249,296,320],"calibration":[6,322],"of":[7,53,108,127,188,256,267,323],"Decision":[8],"System":[10,61,215,279],"One":[11,62,216,280],"models":[12,63,96,281],"on":[13,285,293,307,311],"rheumatology":[14,27,286],"examination":[15,287,312],"questions":[16,28,109],"diagnostic":[18,43,294,308,324],"vignettes.":[19],"Methods:":[20],"Two":[21],"benchmarks":[22],"were":[23,83,190,258,304],"analysed:":[24],"172":[25],"text-based":[26],"from":[29,45,226],"Spanish":[30,37,145],"specialty-training":[31],"examinations":[32],"(expanded":[33],"RheumaMIR":[34,236],"dataset)":[35],"in":[36,112,133,144,148,154,163,235,273],"English,":[39,149],"70":[41],"English":[42],"vignettes":[44,295,309],"RheumBench,":[46,194],"each":[47],"with":[48,85,129,182,252],"5-18":[49],"candidate":[50],"diagnoses,":[51],"52":[52],"which":[54,113],"included":[55,123],"correct":[57],"reference":[58],"diagnosis.":[59],"Four":[60],"(i.e.,":[64],"Jev":[65,209],"1.13,":[66],"Clef,":[67],"Microsoft-Decision-1,":[68],"Laya)":[69],"GPT-6":[71,86,151,159,201,204,219,264,299],"Luna,":[72],"evaluated":[73],"as":[74],"a":[75,88],"decision":[76],"model":[77,217],"through":[78],"OpenAI":[80],"Decisions":[81],"API,":[82],"compared":[84],"Sol,":[87,220],"frontier":[89],"generative":[90,95],"model,":[91],"two":[93],"local":[94],"(Gemma":[97],"4":[98],"12B,":[99],"Qwen3-8B).":[100],"The":[101,277],"primary":[102],"outcome":[103],"was":[104,142,196],"accuracy,":[105],"proportion":[107],"or":[110],"cases":[111],"selected":[115],"option":[116],"matched":[117,232],"answer":[119],"key;":[120],"secondary":[121],"outcomes":[122],"calibration,":[124],"discrimination,":[125],"answers":[128,181,251],"probability":[130,183,245,253],"\u22650.90":[131,184,254],"and,":[132],"RheumaMIR,":[134,139],"language":[135,178],"differences.":[136,179],"Results:":[137],"In":[138,193],"Jev\u2019s":[140,180,301],"93.0%":[143],"91.9%":[147,167],"below":[150,218,298],"Sol":[152,202],"(99.4%":[153],"both":[155,164,274],"languages;":[156],"Holm-adjusted":[157,221],"P\u22640.005);":[158],"Luna":[160,205,265],"reached":[161],"97.7%":[162],"languages,":[165],"Clef":[166,207],"93.0%,":[169],"Microsoft-Decision-1":[171,212],"87.8%":[172],"86.0%,":[174],"respectively,":[175],"without":[176],"detectable":[177],"(76%":[185],"77%":[187],"answers)":[189],"98.5%":[191],"correct.":[192],"lower":[197],"for":[198,263,317],"every":[199],"model:":[200],"76.9%,":[203],"63.5%,":[206],"59.6%,":[208],"50.0%":[210],"40.4%":[213],"(each":[214],"P\u22640.045),":[222],"against":[223,261],"9.6%":[224],"expected":[225],"random":[227],"guessing.":[228],"Jev,":[229],"whose":[230],"probabilities":[231],"its":[233,250],"(over/underconfidence":[237],"index":[238,247],"\u22120.015":[239],"0.000),":[241],"became":[242],"overconfident":[243],"(mean":[244],"0.659;":[246],"+0.159),":[248],"(25%":[255],"cases)":[257],"84.6%":[259],"correct,":[260],"95.5%":[262],"(42%":[266],"cases).":[268],"Laya":[269],"performed":[270],"near":[271],"chance":[272],"benchmarks.":[275],"Conclusion:":[276],"best-performing":[278],"achieved":[282],"high":[283],"questions,":[288,313],"but":[289],"their":[290],"performance":[291],"declined":[292],"remained":[297],"Sol.":[300],"high-confidence":[302],"predictions":[303],"less":[305],"reliable":[306],"than":[310],"highlighting":[314],"need":[316],"task-specific":[318],"evaluation":[319],"improved":[321],"probabilities.":[325]},"counts_by_year":[],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2026-10-29T00:00:00"}