{"id":"https://openalex.org/W7161114435","doi":"https://doi.org/10.48550/arxiv.2605.13542","title":"RealICU: Do LLM Agents Understand Long-Context ICU Data? A Benchmark Beyond Behavior Imitation","display_name":"RealICU: Do LLM Agents Understand Long-Context ICU Data? A Benchmark Beyond Behavior Imitation","publication_year":2026,"publication_date":"2026-05-13","ids":{"openalex":"https://openalex.org/W7161114435","doi":"https://doi.org/10.48550/arxiv.2605.13542"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.13542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.13542","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064753968","display_name":"Chengzhi Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127232","display_name":"Kayseri E\u011fitim ve Ara\u015ft\u0131rma Hastanesi","ror":"https://ror.org/02ysppy04","country_code":"TR","type":"healthcare","lineage":["https://openalex.org/I4210127232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Shen, Chengzhi","raw_affiliation_strings":["Cherise"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cherise","institution_ids":["https://openalex.org/I4210127232"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5130722573","display_name":"Weixiang Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127232","display_name":"Kayseri E\u011fitim ve Ara\u015ft\u0131rma Hastanesi","ror":"https://ror.org/02ysppy04","country_code":"TR","type":"healthcare","lineage":["https://openalex.org/I4210127232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Shen, Weixiang","raw_affiliation_strings":["Cherise"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cherise","institution_ids":["https://openalex.org/I4210127232"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093917855","display_name":"Tobias Susetzky","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127232","display_name":"Kayseri E\u011fitim ve Ara\u015ft\u0131rma Hastanesi","ror":"https://ror.org/02ysppy04","country_code":"TR","type":"healthcare","lineage":["https://openalex.org/I4210127232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Susetzky, Tobias","raw_affiliation_strings":["Cherise"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cherise","institution_ids":["https://openalex.org/I4210127232"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136187338","display_name":"Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127232","display_name":"Kayseri E\u011fitim ve Ara\u015ft\u0131rma Hastanesi","ror":"https://ror.org/02ysppy04","country_code":"TR","type":"healthcare","lineage":["https://openalex.org/I4210127232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Chen","raw_affiliation_strings":["Cherise"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cherise","institution_ids":["https://openalex.org/I4210127232"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136187338","display_name":"Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210127232","display_name":"Kayseri E\u011fitim ve Ara\u015ft\u0131rma Hastanesi","ror":"https://ror.org/02ysppy04","country_code":"TR","type":"healthcare","lineage":["https://openalex.org/I4210127232"]}],"countries":["TR"],"is_corresponding":false,"raw_author_name":"Chen","raw_affiliation_strings":["Cherise"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cherise","institution_ids":["https://openalex.org/I4210127232"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136144929","display_name":"Jun Li","orcid":"https://orcid.org/0000-0003-1613-9448"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Jun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136181870","display_name":"Yuyuan Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yuyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136164807","display_name":"Xuepeng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xuepeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103261260","display_name":"Zhenyu Gong","orcid":"https://orcid.org/0009-0001-0874-5969"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gong, Zhenyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136161741","display_name":"Daniel Rueckert","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rueckert, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136174190","display_name":"Jiazhen Pan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pan, Jiazhen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.5870000123977661,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.5870000123977661,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.11349999904632568,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12574","display_name":"Clinical Reasoning and Diagnostic Skills","score":0.06199999898672104,"subfield":{"id":"https://openalex.org/subfields/2714","display_name":"Family Practice"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hindsight-bias","display_name":"Hindsight bias","score":0.7067999839782715},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6265000104904175},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.580299973487854},{"id":"https://openalex.org/keywords/intensive-care","display_name":"Intensive care","score":0.4496999979019165},{"id":"https://openalex.org/keywords/debiasing","display_name":"Debiasing","score":0.41029998660087585},{"id":"https://openalex.org/keywords/testbed","display_name":"Testbed","score":0.40709999203681946},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.3758000135421753}],"concepts":[{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.7067999839782715},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6265000104904175},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.580299973487854},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.551800012588501},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4657999873161316},{"id":"https://openalex.org/C2987404301","wikidata":"https://www.wikidata.org/wiki/Q679690","display_name":"Intensive care","level":2,"score":0.4496999979019165},{"id":"https://openalex.org/C2779458634","wikidata":"https://www.wikidata.org/wiki/Q24963715","display_name":"Debiasing","level":2,"score":0.41029998660087585},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.40709999203681946},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37709999084472656},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3758000135421753},{"id":"https://openalex.org/C2779328685","wikidata":"https://www.wikidata.org/wiki/Q1475557","display_name":"Patient safety","level":3,"score":0.3476000130176544},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3325999975204468},{"id":"https://openalex.org/C126388530","wikidata":"https://www.wikidata.org/wiki/Q1131737","display_name":"Imitation","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.28690001368522644},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.28139999508857727},{"id":"https://openalex.org/C2776376669","wikidata":"https://www.wikidata.org/wiki/Q5094647","display_name":"Intensive care unit","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.25780001282691956},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2549000084400177},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.2531000077724457},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25270000100135803},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.13542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.13542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.13542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.8110134601593018,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Intensive":[0],"care":[1],"units":[2],"(ICU)":[3],"generate":[4],"long,":[5],"dense":[6],"and":[7,51,60,117,132,144,176,214],"evolving":[8],"streams":[9],"of":[10,55,74,183],"clinical":[11,174],"information,":[12],"where":[13,93],"physicians":[14,99],"must":[15],"repeatedly":[16],"reassess":[17],"patient":[18,58,103],"states":[19],"under":[20,48,89],"time":[21],"pressure,":[22],"underscoring":[23],"a":[24,80,152,170,208],"clear":[25],"need":[26],"for":[27,83,173,212],"reliable":[28],"AI":[29,75,216],"decision":[30],"support.":[31],"Existing":[32,157],"ICU":[33,91],"benchmarks":[34],"typically":[35],"treat":[36],"historical":[37],"clinician":[38],"actions":[39,45,120],"as":[40],"ground":[41],"truth.":[42],"However,":[43],"these":[44],"are":[46,95],"made":[47],"incomplete":[49],"information":[50],"limited":[52],"temporal":[53],"context":[54],"the":[56,70,101,184],"underlying":[57],"state,":[59],"may":[61],"therefore":[62],"be":[63],"suboptimal,":[64],"making":[65],"it":[66],"difficult":[67],"to":[68,180,190],"assess":[69,110],"true":[71],"reasoning":[72,197],"capabilities":[73],"systems.":[76],"We":[77,105,125,186],"introduce":[78,188],"RealICU,":[79,165],"hindsight-annotated":[81],"benchmark":[82],"evaluating":[84],"large":[85],"language":[86],"models":[87],"(LLMs)":[88],"realistic":[90],"conditions,":[92],"labels":[94],"created":[96],"after":[97],"senior":[98],"review":[100],"full":[102],"trajectory.":[104],"formulate":[106],"four":[107],"physician-motivated":[108],"tasks:":[109],"Patient":[111],"Status,":[112],"Acute":[113],"Problems,":[114],"Recommended":[115],"Actions,":[116],"Red":[118],"Flag":[119],"that":[121,194],"risk":[122],"unsafe":[123],"outcomes.":[124],"partition":[126],"each":[127],"trajectory":[128],"with":[129,137,146],"30-min":[130],"windows":[131,148],"release":[133],"two":[134,167],"datasets:":[135],"RealICU-Gold":[136],"930-window":[138],"annotations":[139],"from":[140],"94":[141],"MIMIC-IV":[142],"patients,":[143],"RealICU-Scale":[145],"11,862":[147],"extended":[149],"by":[150],"Oracle,":[151],"physician-validated":[153],"LLM":[154],"hindsight":[155],"labeler.":[156],"LLMs":[158],"including":[159],"memory-augmented":[160],"ones":[161],"performed":[162],"poorly":[163],"on":[164],"exposing":[166],"failure":[168],"modes:":[169],"recall-safety":[171],"tradeoff":[172],"recommendations,":[175],"an":[177],"anchoring":[178],"bias":[179],"early":[181],"interpretations":[182],"patient.":[185],"further":[187],"ICU-Evo":[189],"study":[191],"structured-memory":[192],"agents":[193],"improves":[195],"long-horizon":[196],"but":[198],"does":[199],"not":[200],"fully":[201],"eliminate":[202],"safety":[203],"failures.":[204],"Together,":[205],"RealICU":[206],"provides":[207],"clinically":[209],"grounded":[210],"testbed":[211],"measuring":[213],"improving":[215],"sequential":[217],"decision-support":[218],"in":[219],"high-stakes":[220],"care.":[221],"Project":[222],"page:":[223],"https://chengzhi-leo.github.io/RealICU-Bench/":[224]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-05-15T00:00:00"}