{"id":"https://openalex.org/W4386660952","doi":"https://doi.org/10.1093/eurheartj/ehab724.3052","title":"Inter operator variability of machine learning researchers predicting all-cause mortality in patients admitted to intensive care unit","display_name":"Inter operator variability of machine learning researchers predicting all-cause mortality in patients admitted to intensive care unit","publication_year":2021,"publication_date":"2021-10-01","ids":{"openalex":"https://openalex.org/W4386660952","doi":"https://doi.org/10.1093/eurheartj/ehab724.3052"},"language":"en","primary_location":{"id":"doi:10.1093/eurheartj/ehab724.3052","doi":"https://doi.org/10.1093/eurheartj/ehab724.3052","is_oa":true,"landing_page_url":"https://doi.org/10.1093/eurheartj/ehab724.3052","pdf_url":"https://academic.oup.com/eurheartj/article-pdf/42/Supplement_1/ehab724.3052/41056419/ehab724.3052.pdf","source":{"id":"https://openalex.org/S181568219","display_name":"European Heart Journal","issn_l":"0195-668X","issn":["0195-668X","1522-9645"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"European Heart Journal","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://academic.oup.com/eurheartj/article-pdf/42/Supplement_1/ehab724.3052/41056419/ehab724.3052.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036704435","display_name":"Yola Jones","orcid":"https://orcid.org/0000-0003-0169-6364","observed_orcids":["https://orcid.org/0000-0003-0169-6364"]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Y Jones","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048568770","display_name":"Jennifer Cleland","orcid":"https://orcid.org/0000-0003-1433-9323","observed_orcids":["https://orcid.org/0000-0003-1433-9323"]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"J Cleland","raw_affiliation_strings":["Robertson Centre for Biostatistics, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robertson Centre for Biostatistics, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021111671","display_name":"C Li","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"C Li","raw_affiliation_strings":["University of Glasgow, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Glasgow, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042761326","display_name":"Pierpaolo Pellicori","orcid":"https://orcid.org/0000-0001-7175-0464","observed_orcids":["https://orcid.org/0000-0001-7175-0464"]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"P Pellicori","raw_affiliation_strings":["Robertson Centre for Biostatistics, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robertson Centre for Biostatistics, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039245759","display_name":"Jocelyn M. Friday","orcid":"https://orcid.org/0000-0002-1658-3883","observed_orcids":["https://orcid.org/0000-0002-1658-3883"]},"institutions":[{"id":"https://openalex.org/I7882870","display_name":"University of Glasgow","ror":"https://ror.org/00vtgdb53","country_code":"GB","type":"education","lineage":["https://openalex.org/I7882870"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"J Friday","raw_affiliation_strings":["Robertson Centre for Biostatistics, Glasgow, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Robertson Centre for Biostatistics, Glasgow, United Kingdom","institution_ids":["https://openalex.org/I7882870"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I7882870"],"apc_list":{"value":3918,"currency":"EUR","value_usd":4592},"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.24731866,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"42","issue":"Supplement_1","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9613999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9613999724388123,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.030899999663233757,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10218","display_name":"Sepsis Diagnosis and Treatment","score":0.0019000000320374966,"subfield":{"id":"https://openalex.org/subfields/2713","display_name":"Epidemiology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/all-cause-mortality-prediction","display_name":"all-cause mortality prediction","score":0.765999972820282},{"id":"https://openalex.org/keywords/intensive-care-unit","display_name":"intensive care unit","score":0.656000018119812},{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.8460000157356262},{"id":"https://openalex.org/keywords/machine-learning","display_name":"machine learning","score":0.4050000011920929},{"id":"https://openalex.org/keywords/model-sharing","display_name":"model sharing","score":0.43700000643730164},{"id":"https://openalex.org/keywords/algorithmic-transparency","display_name":"algorithmic transparency","score":0.3880000114440918},{"id":"https://openalex.org/keywords/recurrent-neural-networks","display_name":"recurrent neural networks","score":0.7440000176429749},{"id":"https://openalex.org/keywords/mimic-iii","display_name":"MIMIC-III","score":0.9139999747276306},{"id":"https://openalex.org/keywords/inter-operator-variability","display_name":"inter-operator variability","score":0.75},{"id":"https://openalex.org/keywords/clinical-decision-support","display_name":"clinical decision support","score":0.4860000014305115},{"id":"https://openalex.org/keywords/reporting-quality","display_name":"reporting quality","score":0.4189999997615814},{"id":"https://openalex.org/keywords/clinical-audit","display_name":"clinical audit","score":0.859000027179718}],"concepts":[{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.866126537322998},{"id":"https://openalex.org/C2776376669","wikidata":"https://www.wikidata.org/wiki/Q5094647","display_name":"Intensive care unit","level":2,"score":0.6302825212478638},{"id":"https://openalex.org/C58471807","wikidata":"https://www.wikidata.org/wiki/Q327120","display_name":"Receiver operating characteristic","level":2,"score":0.5879290103912354},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5444445013999939},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5026311874389648},{"id":"https://openalex.org/C160735492","wikidata":"https://www.wikidata.org/wiki/Q31207","display_name":"Health care","level":2,"score":0.4591395854949951},{"id":"https://openalex.org/C194828623","wikidata":"https://www.wikidata.org/wiki/Q2861470","display_name":"Emergency medicine","level":1,"score":0.35856229066848755},{"id":"https://openalex.org/C177713679","wikidata":"https://www.wikidata.org/wiki/Q679690","display_name":"Intensive care medicine","level":1,"score":0.2972176969051361},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.1765848994255066},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.16539043188095093},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1093/eurheartj/ehab724.3052","doi":"https://doi.org/10.1093/eurheartj/ehab724.3052","is_oa":true,"landing_page_url":"https://doi.org/10.1093/eurheartj/ehab724.3052","pdf_url":"https://academic.oup.com/eurheartj/article-pdf/42/Supplement_1/ehab724.3052/41056419/ehab724.3052.pdf","source":{"id":"https://openalex.org/S181568219","display_name":"European Heart Journal","issn_l":"0195-668X","issn":["0195-668X","1522-9645"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"European Heart Journal","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1093/eurheartj/ehab724.3052","doi":"https://doi.org/10.1093/eurheartj/ehab724.3052","is_oa":true,"landing_page_url":"https://doi.org/10.1093/eurheartj/ehab724.3052","pdf_url":"https://academic.oup.com/eurheartj/article-pdf/42/Supplement_1/ehab724.3052/41056419/ehab724.3052.pdf","source":{"id":"https://openalex.org/S181568219","display_name":"European Heart Journal","issn_l":"0195-668X","issn":["0195-668X","1522-9645"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["anvur-scientific","cwts-core","dongbi-a","doyens","jufo-3","ki-jl-3","medline","norway-2","poland-200","russia-white-list-1","vabb-shw"],"host_organization":"https://openalex.org/P4310311648","host_organization_name":"Oxford University Press","host_organization_lineage":["https://openalex.org/P4310311648","https://openalex.org/P4310311647"],"host_organization_lineage_names":["Oxford University Press","University of Oxford"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"European Heart Journal","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.9537000060081482,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"sustainable_development_goals_aurora":[{"score":0.8600000143051147,"id":"https://metadata.un.org/sdg/3","display_name":"Good health and well-being"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386660952.pdf","grobid_xml":"https://content.openalex.org/works/W4386660952.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2788104422","https://openalex.org/W4380136483","https://openalex.org/W7126397732","https://openalex.org/W4302023680","https://openalex.org/W7131641617","https://openalex.org/W3121101121","https://openalex.org/W7165903494","https://openalex.org/W2969097171","https://openalex.org/W3153037631","https://openalex.org/W4225326659"],"abstract_inverted_index":{"Abstract":[0],"Background":[1],"The":[2,294],"number":[3],"of":[4,17,62,71,82,125,197,209,296,303,423],"publications":[5],"using":[6,232],"machine":[7],"learning":[8],"(ML)":[9],"to":[10,36,43,46,119,229,347,377,414],"predict":[11],"cardiovascular":[12],"outcomes":[13],"and":[14,85,114,123,150,162,175,182,249,265,286,288,313,358,395],"identify":[15],"clusters":[16],"patients":[18,262],"at":[19],"greater":[20],"risk":[21],"has":[22],"risen":[23],"dramatically":[24],"in":[25,51,75,148,173,201,283,417],"recent":[26],"years.":[27],"However,":[28],"research":[29,155],"papers":[30],"which":[31,157,280],"use":[32],"ML":[33,65,174,351,416],"often":[34],"fail":[35],"provide":[37],"sufficient":[38],"information":[39,72,96,326],"about":[40,329,368],"their":[41,187,369],"algorithms":[42,66,215],"enable":[44],"results":[45,63,333,341,354,398],"be":[47,318,384],"replicated":[48],"by":[49,205],"others":[50],"the":[52,60,83,106,159,179,210,213,235,297,304,310,314,332,335,339,393,396,402],"same":[53,160,180],"or":[54,253],"different":[55,69],"datasets.":[56],"Aim":[57],"To":[58],"test":[59],"reproducibility":[61],"from":[64,97,105,143,152,216,260,334],"given":[67,178,320],"three":[68,133,223],"levels":[70],"commonly":[73],"found":[74,400],"publications:":[76],"model":[77,165,198,321],"type":[78,196,322,372],"alone,":[79],"a":[80,91,126,144,206,289],"description":[81,208],"model,":[84],"complete":[86,164,214,403],"algorithm.":[87],"Methods":[88],"MIMIC-III":[89],"is":[90,117,361,374,390,405],"healthcare":[92],"dataset":[93,161,181],"comprising":[94],"detailed":[95],"over":[98],"60,000":[99],"intensive":[100],"care":[101],"unit":[102],"(ICU)":[103],"admissions":[104,258],"Beth":[107],"Israel":[108],"Deaconess":[109],"Medical":[110],"Centre":[111],"between":[112,263],"2001":[113,264],"2012.":[115],"Access":[116],"available":[118],"everyone":[120],"pending":[121],"approval":[122],"completion":[124],"short":[127],"training":[128],"course.":[129],"Using":[130],"this":[131,190,389],"dataset,":[132],"models":[134,226,231,272,337,382],"for":[135],"predicting":[136],"all-cause":[137],"in-hospital":[138,269],"mortality":[139],"were":[140,219,227,273],"created,":[141],"two":[142,275],"PhD":[145,171],"student":[146,172],"working":[147],"ML,":[149],"one":[151],"an":[153],"existing":[154],"paper":[156],"used":[158],"provided":[163,328],"information.":[166],"A":[167],"second":[168,191,311],"researcher":[169,192],"(a":[170],"cardiology)":[176],"was":[177,183,193,199,327],"tasked":[184],"with":[185,251,353,386],"reproducing":[186],"results.":[188],"Initially,":[189],"told":[194],"what":[195],"created":[200],"each":[202,217],"case,":[203,394],"followed":[204],"brief":[207],"algorithms.":[211],"Finally,":[212],"participant":[218],"provided.":[220],"In":[221,345],"all":[222],"scenarios,":[224],"recreated":[225,385],"compared":[228],"original":[230,305,340],"Area":[233],"Under":[234],"Receiver":[236],"Operating":[237],"Characteristic":[238],"Curve":[239],"(AUC).":[240],"Results":[241],"After":[242],"excluding":[243],"those":[244],"younger":[245],"than":[246],"18":[247],"years":[248],"events":[250],"missing":[252],"invalid":[254],"entries,":[255],"21,139":[256],"ICU":[257],"remained":[259],"18,094":[261],"2012,":[266],"including":[267],"2,797":[268],"deaths.":[270],"Three":[271],"produced:":[274],"Recurrent":[276],"Neural":[277],"Networks":[278],"(RNNs)":[279],"differed":[281],"significantly":[282],"internal":[284],"weights":[285],"variables,":[287],"Boosted":[290],"Tree":[291],"Classifier":[292],"(BTC).":[293],"AUC":[295],"first":[298],"reproduced":[299,319,336],"RNN":[300,306,312],"matched":[301,338],"that":[302,355,363],"(Figure":[307],"1),":[308],"however":[309],"BTC":[315],"could":[316],"not":[317,375,391],"alone.":[323],"As":[324],"more":[325,342],"these":[330],"algorithms,":[331],"closely.":[343],"Conclusions":[344],"order":[346],"create":[348],"clinically":[349],"useful":[350],"tools":[352],"are":[356,399],"reproducible":[357],"consistent,":[359],"it":[360],"vital":[362],"researchers":[364],"share":[365],"enough":[366,376],"detail":[367],"models.":[370],"Model":[371],"alone":[373],"guarantee":[378],"reproducibility.":[379],"Although":[380],"some":[381],"can":[383],"limited":[387],"information,":[388],"always":[392],"best":[397],"when":[401,412],"algorithm":[404],"shared.":[406],"These":[407],"findings":[408],"have":[409],"huge":[410],"relevance":[411],"trying":[413],"apply":[415],"clinical":[418],"practice.":[419],"Funding":[420],"Acknowledgement":[421],"Type":[422],"funding":[424],"sources:":[425],"None.":[426]},"counts_by_year":[],"updated_date":"2026-10-27T15:51:10.671819","created_date":"2023-09-13T00:00:00"}