{"id":"https://openalex.org/W7135231172","doi":"https://doi.org/10.48550/arxiv.2603.11413","title":"Evaluation format, not model capability, drives triage failure in the assessment of consumer health AI","display_name":"Evaluation format, not model capability, drives triage failure in the assessment of consumer health AI","publication_year":2026,"publication_date":"2026-03-12","ids":{"openalex":"https://openalex.org/W7135231172","doi":"https://doi.org/10.48550/arxiv.2603.11413"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.11413","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.11413","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053537164","display_name":"David Fraile Navarro","orcid":"https://orcid.org/0000-0002-1108-7071"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Navarro, David Fraile","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129032742","display_name":"Farah Magrabi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Magrabi, Farah","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5088581420","display_name":"Enrico Coiera","orcid":"https://orcid.org/0000-0002-6444-6584"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Coiera, Enrico","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8738999962806702,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8738999962806702,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11519","display_name":"Digital Mental Health Interventions","score":0.0215000007301569,"subfield":{"id":"https://openalex.org/subfields/3202","display_name":"Applied Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.008700000122189522,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/triage","display_name":"Triage","score":0.8931999802589417},{"id":"https://openalex.org/keywords/headline","display_name":"Headline","score":0.7775999903678894},{"id":"https://openalex.org/keywords/health-care","display_name":"Health care","score":0.38350000977516174},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.3792000114917755},{"id":"https://openalex.org/keywords/heartbeat","display_name":"Heartbeat","score":0.35929998755455017},{"id":"https://openalex.org/keywords/blackout","display_name":"Blackout","score":0.3330000042915344},{"id":"https://openalex.org/keywords/patient-safety","display_name":"Patient safety","score":0.29420000314712524}],"concepts":[{"id":"https://openalex.org/C2777120189","wikidata":"https://www.wikidata.org/wiki/Q780067","display_name":"Triage","level":2,"score":0.8931999802589417},{"id":"https://openalex.org/C2778689934","wikidata":"https://www.wikidata.org/wiki/Q1313396","display_name":"Headline","level":2,"score":0.7775999903678894},{"id":"https://openalex.org/C545542383","wikidata":"https://www.wikidata.org/wiki/Q2751242","display_name":"Medical emergency","level":1,"score":0.4918000102043152},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.41110000014305115},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.38350000977516174},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.3792000114917755},{"id":"https://openalex.org/C13852961","wikidata":"https://www.wikidata.org/wiki/Q17021880","display_name":"Heartbeat","level":2,"score":0.35929998755455017},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3472000062465668},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3434000015258789},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3346000015735626},{"id":"https://openalex.org/C2777693866","wikidata":"https://www.wikidata.org/wiki/Q359099","display_name":"Blackout","level":4,"score":0.3330000042915344},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.3009999990463257},{"id":"https://openalex.org/C2779328685","wikidata":"https://www.wikidata.org/wiki/Q1475557","display_name":"Patient safety","level":3,"score":0.29420000314712524},{"id":"https://openalex.org/C62555980","wikidata":"https://www.wikidata.org/wiki/Q1460420","display_name":"Emergency management","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.2743000090122223},{"id":"https://openalex.org/C2778707766","wikidata":"https://www.wikidata.org/wiki/Q202064","display_name":"Phone","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C2777338717","wikidata":"https://www.wikidata.org/wiki/Q1762621","display_name":"Vendor","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.2662999927997589},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.2644999921321869},{"id":"https://openalex.org/C90673727","wikidata":"https://www.wikidata.org/wiki/Q901718","display_name":"Product (mathematics)","level":2,"score":0.2526000142097473}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.11413","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.11413","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.11413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Ramaswamy":[0],"et":[1],"al.":[2],"reported":[3],"in":[4,114,159],"Nature":[5],"Medicine":[6],"that":[7,15,41,187,220],"ChatGPT":[8],"Health":[9],"under-triages":[10],"51.6%":[11],"of":[12,37,116,206,212],"emergencies,":[13],"concluding":[14],"consumer-facing":[16],"AI":[17,215],"triage":[18,100,124,208],"poses":[19],"safety":[20],"risks.":[21],"However,":[22],"their":[23,160],"evaluation":[24,196,211],"used":[25],"an":[26],"exam-style":[27],"protocol":[28],"--":[29,40],"forced":[30,131,144],"A/B/C/D":[31,132],"output,":[32],"knowledge":[33],"suppression,":[34],"and":[35,79,89,121,198],"suppression":[36],"clarifying":[38],"questions":[39],"differs":[42],"fundamentally":[43],"from":[44,126],"how":[45],"consumers":[46],"use":[47],"health":[48,214],"chatbots.":[49],"We":[50],"tested":[51],"five":[52],"frontier":[53],"LLMs":[54],"(GPT-5.2,":[55],"Claude":[56,59],"Sonnet":[57],"4.6,":[58,61],"Opus":[60],"Gemini":[62,65],"3":[63],"Flash,":[64],"3.1":[66],"Pro)":[67],"on":[68,171,195],"a":[69,203],"17-scenario":[70],"partial":[71],"replication":[72],"bank":[73],"under":[74,218],"constrained":[75],"(exam-style,":[76],"1,275":[77],"trials)":[78,84],"naturalistic":[80],"(patient-style":[81],"messages,":[82],"850":[83],"conditions,":[85],"with":[86,143,148],"targeted":[87],"ablations":[88],"prompt-faithful":[90],"checks":[91,170],"using":[92],"the":[93,135,164,172,178,188],"authors'":[94,173],"released":[95,175],"prompts.":[96],"Naturalistic":[97],"interaction":[98],"improved":[99,125],"accuracy":[101],"by":[102],"6.4":[103],"percentage":[104],"points":[105],"($p":[106],"=":[107],"0.015$).":[108],"Diabetic":[109],"ketoacidosis":[110],"was":[111,134],"correctly":[112],"triaged":[113],"100%":[115,147],"trials":[117],"across":[118],"all":[119],"models":[120,140],"conditions.":[122],"Asthma":[123],"48%":[127],"to":[128],"80%.":[129],"The":[130],"format":[133,166,197],"dominant":[136],"failure":[137],"mechanism:":[138],"three":[139],"scored":[141],"0--24%":[142],"choice":[145],"but":[146],"free":[149],"text":[150],"(all":[151],"$p":[152],"<":[153],"10^{-8}$),":[154],"consistently":[155],"recommending":[156],"emergency":[157],"care":[158],"own":[161],"words":[162],"while":[163],"forced-choice":[165],"registered":[167],"under-triage.":[168],"Prompt-faithful":[169],"exact":[174],"prompts":[176],"confirmed":[177],"scaffold":[179],"produces":[180],"model-dependent,":[181],"case-dependent":[182],"results.":[183],"Our":[184],"results":[185],"suggest":[186],"headline":[189],"under-triage":[190],"rate":[191],"is":[192],"highly":[193],"contingent":[194],"may":[199],"not":[200],"generalize":[201],"as":[202],"stable":[204],"estimate":[205],"deployed":[207],"behavior.":[209],"Valid":[210],"consumer":[213],"requires":[216],"testing":[217],"conditions":[219],"reflect":[221],"actual":[222],"use.":[223]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-14T00:00:00"}