{"id":"https://openalex.org/W7166176728","doi":"https://doi.org/10.1007/s10916-026-02430-0","title":"Calibration of Self-Reported Confidence and Accuracy of Large Language Models in Medical Question Answering","display_name":"Calibration of Self-Reported Confidence and Accuracy of Large Language Models in Medical Question Answering","publication_year":2026,"publication_date":"2026-06-26","ids":{"openalex":"https://openalex.org/W7166176728","doi":"https://doi.org/10.1007/s10916-026-02430-0","pmid":"https://pubmed.ncbi.nlm.nih.gov/42360536"},"language":"en","primary_location":{"id":"doi:10.1007/s10916-026-02430-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10916-026-02430-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10916-026-02430-0.pdf","source":{"id":"https://openalex.org/S37151855","display_name":"Journal of Medical Systems","issn_l":"0148-5598","issn":["0148-5598","1573-689X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10916-026-02430-0.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031763874","display_name":"Sebastian Daniel Boie","orcid":"https://orcid.org/0000-0001-7993-7901"},"institutions":[{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]},{"id":"https://openalex.org/I7877124","display_name":"Charit\u00e9 - Universit\u00e4tsmedizin Berlin","ror":"https://ror.org/001w7jn25","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I7877124"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Sebastian Daniel Boie","raw_affiliation_strings":["Department of Anesthesiology and Intensive Care Medicine (CCM/CVK), Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Berlin, Germany. sebastian-daniel.boie@charite.de","Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany. sebastian-daniel.boie@charite.de"],"raw_orcid":"https://orcid.org/0000-0001-7993-7901","affiliations":[{"raw_affiliation_string":"Department of Anesthesiology and Intensive Care Medicine (CCM/CVK), Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Berlin, Germany. sebastian-daniel.boie@charite.de","institution_ids":["https://openalex.org/I39343248"]},{"raw_affiliation_string":"Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany. sebastian-daniel.boie@charite.de","institution_ids":["https://openalex.org/I39343248","https://openalex.org/I7877124"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093933525","display_name":"Florian Reis","orcid":"https://orcid.org/0009-0009-1064-9173"},"institutions":[{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Florian Reis","raw_affiliation_strings":["Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany","institution_ids":["https://openalex.org/I39343248"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103110542","display_name":"Nicolas Frey","orcid":"https://orcid.org/0009-0000-0195-6096"},"institutions":[{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Nicolas Frey","raw_affiliation_strings":["Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany","institution_ids":["https://openalex.org/I39343248"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074511313","display_name":"Elias Gr\u00fcnewald","orcid":"https://orcid.org/0000-0001-9076-9240"},"institutions":[{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Elias Gr\u00fcnewald","raw_affiliation_strings":["Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany","institution_ids":["https://openalex.org/I39343248"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5136664287","display_name":"Felix Balzer","orcid":null},"institutions":[{"id":"https://openalex.org/I39343248","display_name":"Humboldt-Universit\u00e4t zu Berlin","ror":"https://ror.org/01hcx6992","country_code":"DE","type":"education","lineage":["https://openalex.org/I39343248"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Felix Balzer","raw_affiliation_strings":["Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Medical Informatics, Charit\u00e9 - University Medical Center Berlin, corporate member of Freie Universit\u00e4t Berlin and Humboldt-Universit\u00e4t zu Berlin, Charit\u00e9platz 1, 10117, Berlin, Germany","institution_ids":["https://openalex.org/I39343248"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5031763874"],"corresponding_institution_ids":["https://openalex.org/I39343248","https://openalex.org/I7877124"],"apc_list":{"value":3390,"currency":"EUR","value_usd":4390},"apc_paid":{"value":3390,"currency":"EUR","value_usd":4390},"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.89066272,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"50","issue":"1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7767999768257141,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.7767999768257141,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.03229999914765358,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.024000000208616257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/overconfidence-effect","display_name":"Overconfidence effect","score":0.864799976348877},{"id":"https://openalex.org/keywords/calibration","display_name":"Calibration","score":0.7071999907493591},{"id":"https://openalex.org/keywords/confidence-interval","display_name":"Confidence interval","score":0.6371999979019165},{"id":"https://openalex.org/keywords/sonnet","display_name":"Sonnet","score":0.5218999981880188},{"id":"https://openalex.org/keywords/interpretability","display_name":"Interpretability","score":0.3758000135421753},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.3483999967575073}],"concepts":[{"id":"https://openalex.org/C51110983","wikidata":"https://www.wikidata.org/wiki/Q16503490","display_name":"Overconfidence effect","level":2,"score":0.864799976348877},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.7071999907493591},{"id":"https://openalex.org/C44249647","wikidata":"https://www.wikidata.org/wiki/Q208498","display_name":"Confidence interval","level":2,"score":0.6371999979019165},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6137999892234802},{"id":"https://openalex.org/C38721330","wikidata":"https://www.wikidata.org/wiki/Q80056","display_name":"Sonnet","level":3,"score":0.5218999981880188},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42989999055862427},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.41929998993873596},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.3758000135421753},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.34850001335144043},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.3483999967575073},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.34220001101493835},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3416999876499176},{"id":"https://openalex.org/C145642194","wikidata":"https://www.wikidata.org/wiki/Q870895","display_name":"Health informatics","level":3,"score":0.3375000059604645},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2924000024795532},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.2874000072479248},{"id":"https://openalex.org/C163763905","wikidata":"https://www.wikidata.org/wiki/Q17075943","display_name":"Precision medicine","level":2,"score":0.2741999924182892},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.25940001010894775}],"mesh":[{"descriptor_ui":"D000098342","descriptor_name":"Large Language Models","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D002138","descriptor_name":"Calibration","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D057566","descriptor_name":"Self Report","qualifier_ui":"Q000592","qualifier_name":"standards","is_major_topic":true}],"locations_count":3,"locations":[{"id":"doi:10.1007/s10916-026-02430-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10916-026-02430-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10916-026-02430-0.pdf","source":{"id":"https://openalex.org/S37151855","display_name":"Journal of Medical Systems","issn_l":"0148-5598","issn":["0148-5598","1573-689X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Systems","raw_type":"journal-article"},{"id":"pmid:42360536","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/42360536","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of medical systems","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:13309378","is_oa":true,"landing_page_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC13309378/","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"J Med Syst","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1007/s10916-026-02430-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10916-026-02430-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10916-026-02430-0.pdf","source":{"id":"https://openalex.org/S37151855","display_name":"Journal of Medical Systems","issn_l":"0148-5598","issn":["0148-5598","1573-689X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Medical Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.47668689489364624,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320321526","display_name":"Charit\u00e9 \u2013 Universit\u00e4tsmedizin Berlin","ror":"https://ror.org/001w7jn25"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166176728.pdf","grobid_xml":"https://content.openalex.org/works/W7166176728.grobid-xml"},"referenced_works_count":13,"referenced_works":["https://openalex.org/W2762658547","https://openalex.org/W3118929067","https://openalex.org/W4396802056","https://openalex.org/W4402426751","https://openalex.org/W4402671618","https://openalex.org/W4403362497","https://openalex.org/W4406256654","https://openalex.org/W4406679533","https://openalex.org/W4410424798","https://openalex.org/W4411121371","https://openalex.org/W4412353632","https://openalex.org/W4412488514","https://openalex.org/W4413062330"],"related_works":[],"abstract_inverted_index":{"OBJECTIVE:":[0],"To":[1],"evaluate":[2],"the":[3],"alignment":[4],"between":[5,61,101,131],"self-reported":[6],"confidence":[7,44],"of":[8,95],"large":[9],"language":[10],"models":[11,147],"(LLM)":[12],"and":[13,35,42,64,81,133],"their":[14],"accuracy":[15,63,109],"in":[16],"answering":[17],"medical":[18,121],"multiple-choice":[19],"questions.":[20],"MATERIALS":[21],"AND":[22],"METHODS:":[23],"We":[24,116,137],"prompted":[25],"six":[26],"LLMs":[27,97],"(GPT-5,":[28],"GPT-5-mini,":[29],"GPT-5-nano,":[30],"GPT-4o,":[31],"Claude":[32],"Sonnet":[33,74],"4.5,":[34],"Gemini":[36],"2.5)":[37],"to":[38],"answer":[39],"MedMCQA":[40],"items":[41],"report":[43],"scores.":[45],"Based":[46],"on":[47],"12,000":[48],"LLM":[49,143],"responses,":[50],"we":[51],"calculated":[52],"Expected":[53],"Calibration":[54],"Errors":[55],"(ECE)":[56],"by":[57,71],"averaging":[58],"absolute":[59],"differences":[60],"observed":[62],"predicted":[65],"confidence.":[66],"RESULTS:":[67],"Mean":[68],"ECE":[69,129],"differed":[70],"model":[72],"(Claude":[73],"4.5":[75],"best:":[76,86],"0.06;":[77],"Gpt-4o":[78],"worst:":[79,92],"0.127)":[80],"varied":[82],"across":[83,120],"specialties":[84],"(\"Skin\"":[85],"0.041;":[87],"\"Social":[88],"&":[89],"Preventive":[90],"Medicine\"":[91],"0.141).":[93],"Accuracy":[94],"examined":[96],"showed":[98],"analogous":[99],"variation":[100],"specialties.":[102],"DISCUSSION:":[103],"Our":[104],"results":[105],"demonstrate":[106],"that":[107],"high":[108],"does":[110],"not":[111],"guarantee":[112],"reliable":[113],"uncertainty":[114],"estimation.":[115],"identified":[117],"substantial":[118],"heterogeneity":[119],"specialties,":[122],"where":[123],"pooled":[124],"metrics":[125],"masked":[126],"a":[127],"threefold":[128],"increase":[130],"best-":[132],"worst-performing":[134],"domains.":[135],"CONCLUSION:":[136],"recommend":[138],"incorporating":[139],"calibration":[140],"reporting":[141],"into":[142],"evaluations,":[144],"as":[145],"larger":[146],"exhibit":[148],"improved":[149],"\"self-knowledge\",":[150],"but":[151],"uneven":[152],"overconfidence":[153],"persists.":[154]},"counts_by_year":[],"updated_date":"2026-06-30T13:55:48.251075","created_date":"2026-06-27T00:00:00"}