{"id":"https://openalex.org/W4411270620","doi":"https://doi.org/10.1109/access.2025.3579631","title":"ML Auditing and Reproducibility: Applying a Core Criteria Catalog to an Early Sepsis Onset Detection System","display_name":"ML Auditing and Reproducibility: Applying a Core Criteria Catalog to an Early Sepsis Onset Detection System","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4411270620","doi":"https://doi.org/10.1109/access.2025.3579631"},"language":"en","primary_location":{"id":"doi:10.1109/access.2025.3579631","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3579631","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2025.3579631","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103189914","display_name":"Markus Schwarz","orcid":"https://orcid.org/0009-0003-0457-0289"},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Markus Schwarz","raw_affiliation_strings":["Medical Data Integration Center (MeDIC LMU), Faculty of Medicine, University Hospital LMU Munich, Planegg, Germany"],"raw_orcid":"https://orcid.org/0009-0003-0457-0289","affiliations":[{"raw_affiliation_string":"Medical Data Integration Center (MeDIC LMU), Faculty of Medicine, University Hospital LMU Munich, Planegg, Germany","institution_ids":["https://openalex.org/I8204097"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008928704","display_name":"Ludwig Christian Hinske","orcid":"https://orcid.org/0000-0001-7273-5899"},"institutions":[{"id":"https://openalex.org/I2802007660","display_name":"University Hospital Augsburg","ror":"https://ror.org/03b0k9c14","country_code":"DE","type":"healthcare","lineage":["https://openalex.org/I2802007660"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ludwig Christian Hinske","raw_affiliation_strings":["Institute for Digital Medicine, Faculty of Medicine, University Hospital Augsburg, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0001-7273-5899","affiliations":[{"raw_affiliation_string":"Institute for Digital Medicine, Faculty of Medicine, University Hospital Augsburg, Munich, Germany","institution_ids":["https://openalex.org/I2802007660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057302175","display_name":"Ulrich Mansmann","orcid":"https://orcid.org/0000-0002-9955-8906"},"institutions":[{"id":"https://openalex.org/I4210099379","display_name":"Zimmer Biomet (Germany)","ror":"https://ror.org/016731f38","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210099379","https://openalex.org/I4210115238"]},{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Ulrich Mansmann","raw_affiliation_strings":["Institute for Medical Information Processing, Biometry and Epidemiology (IBE), Faculty of Medicine, LMU Munich, Munich, Germany"],"raw_orcid":"https://orcid.org/0000-0002-9955-8906","affiliations":[{"raw_affiliation_string":"Institute for Medical Information Processing, Biometry and Epidemiology (IBE), Faculty of Medicine, LMU Munich, Munich, Germany","institution_ids":["https://openalex.org/I8204097","https://openalex.org/I4210099379"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064758815","display_name":"Fady Albashiti","orcid":"https://orcid.org/0000-0002-0671-152X"},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Fady Albashiti","raw_affiliation_strings":["Medical Data Integration Center (MeDIC LMU), Faculty of Medicine, University Hospital LMU Munich, Planegg, Germany"],"raw_orcid":"https://orcid.org/0000-0002-0671-152X","affiliations":[{"raw_affiliation_string":"Medical Data Integration Center (MeDIC LMU), Faculty of Medicine, University Hospital LMU Munich, Planegg, Germany","institution_ids":["https://openalex.org/I8204097"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5103189914"],"corresponding_institution_ids":["https://openalex.org/I8204097"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":2.5159,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.909817,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":98},"biblio":{"volume":"13","issue":null,"first_page":"104899","last_page":"104915"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.1655000001192093,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.1655000001192093,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.15060000121593475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.14959999918937683,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.675779402256012},{"id":"https://openalex.org/keywords/audit","display_name":"Audit","score":0.645933210849762},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6436010003089905},{"id":"https://openalex.org/keywords/core","display_name":"Core (optical fiber)","score":0.46672478318214417},{"id":"https://openalex.org/keywords/medical-physics","display_name":"Medical physics","score":0.3277590870857239},{"id":"https://openalex.org/keywords/accounting","display_name":"Accounting","score":0.21838340163230896},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.20653891563415527},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.16776081919670105},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12020924687385559},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.08442047238349915},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.07875627279281616}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.675779402256012},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.645933210849762},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6436010003089905},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.46672478318214417},{"id":"https://openalex.org/C19527891","wikidata":"https://www.wikidata.org/wiki/Q1120908","display_name":"Medical physics","level":1,"score":0.3277590870857239},{"id":"https://openalex.org/C121955636","wikidata":"https://www.wikidata.org/wiki/Q4116214","display_name":"Accounting","level":1,"score":0.21838340163230896},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.20653891563415527},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.16776081919670105},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12020924687385559},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.08442047238349915},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.07875627279281616}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2025.3579631","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3579631","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:f549557961504208b149a6fdd5f096bf","is_oa":true,"landing_page_url":"https://doaj.org/article/f549557961504208b149a6fdd5f096bf","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 13, Pp 104899-104915 (2025)","raw_type":"article"},{"id":"pmh:oai:share.osf.io:bcd891c6-3289-4bb0-9868-7590a54940b6","is_oa":true,"landing_page_url":"https://osf.io/bca7n","pdf_url":null,"source":{"id":"https://openalex.org/S4306401127","display_name":"OSF Preprints (OSF Preprints)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I2799848540","host_organization_name":"Center for Open Science","host_organization_lineage":["https://openalex.org/I2799848540"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"ProjectComponent"}],"best_oa_location":{"id":"doi:10.1109/access.2025.3579631","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2025.3579631","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":18,"referenced_works":["https://openalex.org/W2035707873","https://openalex.org/W2135046866","https://openalex.org/W2136589719","https://openalex.org/W2150290224","https://openalex.org/W2520567882","https://openalex.org/W2964199361","https://openalex.org/W3174786846","https://openalex.org/W3189603506","https://openalex.org/W4317748910","https://openalex.org/W4385245566","https://openalex.org/W4385762558","https://openalex.org/W4392667179","https://openalex.org/W4399587108","https://openalex.org/W4405439378","https://openalex.org/W6739901393","https://openalex.org/W6745609711","https://openalex.org/W6875683615","https://openalex.org/W6943793429"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W4324138256"],"abstract_inverted_index":{"Background:":[0],"On":[1],"the":[2,44,48,62,71,80,89,130,138,149,159,168,185,207,238,251,254,285,327,330,340,358,364,367,373,378,395,412,422],"way":[3],"towards":[4],"a":[5,19,55,222,299,432],"commonly":[6],"agreed":[7],"framework":[8],"for":[9,372,377,411,446,490],"auditing":[10],"ML":[11,45,132,314,463,499],"algorithms,":[12],"in":[13,54,178,268,298,366],"our":[14,29,65,113,126,198,383,459],"previous":[15],"paper":[16],"we":[17,27,83,212,244,322],"proposed":[18,87],"30-question":[20,127],"core":[21],"criteria":[22],"catalog.":[23],"In":[24,197,250,320],"this":[25,476],"paper,":[26],"apply":[28,61,125,458],"catalog":[30,66,128,293,460],"to":[31,70,123,129,241,270,305,325,344,348,400,416,434,461,487],"an":[32,105,214,257,271,303],"early":[33],"sepsis":[34,49,73,131,186,242,331,417],"onset":[35,418],"detection":[36],"system":[37,51],"use":[38],"case.":[39],"Methods:":[40],"The":[41,171,182,292,361,387],"assessment":[42,195],"of":[43,57,64,93,137,148,158,184,216,224,259,273,313,329,339,363,385,390,394,495],"algorithm":[46,133,190,315],"behind":[47],"prediction":[50,205],"takes":[52],"place":[53],"kind":[56],"external":[58],"audit.":[59],"We":[60,102,120],"questions":[63,139,150,160],"with":[67,264,278],"described":[68],"context":[69],"available":[72],"project":[74,187],"resources":[75],"made":[76],"publicly":[77],"available.":[78],"For":[79,237],"audit":[81],"process":[82],"considered":[84],"three":[85],"steps":[86,338],"by":[88,112,382],"Supreme":[90],"Audit":[91],"Institutions":[92],"Finland":[94],"et":[95],"al.":[96],"and":[97,156,194,221,232,288,308,311,352,357,375,465],"utilized":[98],"inter-rater":[99],"reliability":[100],"techniques.":[101],"also":[103,481],"conducted":[104],"extensive":[106],"reproduction":[107,199,341],"study,":[108,200],"as":[109,142,151,161,430],"being":[110],"encouraged":[111],"catalog,":[114],"including":[115],"data":[116,192,267,402],"perturbation":[117,252],"experiments.":[118],"Results:":[119],"were":[121,140,323],"able":[122,324],"successfully":[124],"development":[134,316,464,483],"project.":[135],"37%":[136],"rated":[141],"<italic":[143,152,162],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[144,153,163],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">fully":[145],"addressed</i>,":[146,165],"33%":[147],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">partially":[154],"addressed</i>":[155],"30%":[157],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">not":[164],"based":[166],"on":[167,189,206,284,354],"first":[169],"auditor.":[170],"weighted":[172,424],"Cohen\u2019s":[173,425],"kappa":[174],"agreement":[175,436],"coefficient":[176,427],"results":[177,295],"\u03ba":[179],"=":[180],"0.51.":[181],"focus":[183],"is":[188,407,428],"design,":[191],"properties":[193],"metrics.":[196,335],"using":[201],"externally":[202],"validated":[203,289],"pooled":[204],"self-attention":[208,396],"deep":[209,397],"learning":[210,398],"model,":[211],"achieved":[213],"AUC":[215,258,272,374,388],"0.717":[217],"(95%":[218,226,261,275],"CI,":[219,227,262,276],"0.693-0.740)":[220],"PPV":[223],"28.3":[225],"24.5-32.0)":[228],"at":[229],"80%":[230],"TPR":[231],"18.8%":[233],"sepsis-case":[234],"prevalence":[235],"harmonization.":[236],"lead":[239,414],"time":[240,415],"onset,":[243],"could":[245,449,473],"not":[246],"reproduce":[247,326],"meaningful":[248],"values.":[249],"experiment,":[253],"model":[255,399],"showed":[256],"0.799":[260],"0.756-0.843)":[263],"modified":[265],"input":[266,280,401],"contrast":[269],"0.788":[274],"0.743-0.833)":[277],"original":[279],"data,":[281],"when":[282],"trained":[283],"AUMC":[286],"dataset":[287],"externally.":[290],"Discussion:":[291],"application":[294,471],"are":[296],"visualized":[297],"radar":[300],"diagram,":[301],"allowing":[302],"auditor":[304],"quickly":[306],"assess":[307],"compare":[309],"strengths":[310],"weaknesses":[312],"or":[317,484],"implementation":[318,466,485],"projects.":[319,467],"general,":[321],"magnitude":[328],"project\u2019s":[332],"reported":[333],"performance":[334],"However,":[336],"certain":[337],"study":[342],"proved":[343],"be":[345,450,474],"challenging":[346],"due":[347],"necessary":[349],"code":[350],"changes":[351],"dependencies":[353],"package":[355],"versions":[356],"runtime":[359],"environment.":[360],"extent":[362],"deviation":[365],"result":[368],"metrics":[369],"was":[370],"\u22125.83%":[371],"\u221211.03%":[376],"PPV,":[379],"presumably":[380],"explained":[381],"absence":[384],"tuning.":[386],"change":[389],"1.45%":[391],"indicates":[392],"resilience":[393],"manipulation.":[403],"An":[404],"algorithmic":[405],"error":[406],"most":[408],"likely":[409],"responsible":[410],"missing":[413],"metric.":[419],"Even":[420],"though":[421],"acquired":[423],"kapa":[426],"interpreted":[429],"having":[431],"\"fair":[433],"good\"":[435],"between":[437],"both":[438],"auditors,":[439],"there":[440],"exists":[441],"potential":[442],"subjectivity":[443],"showing":[444],"room":[445],"improvement.":[447],"This":[448],"mitigated":[451],"if":[452],"more":[453],"groups":[454],"(multiple":[455],"auditors)":[456],"would":[457],"existing":[462],"A":[468],"subsequent":[469],"\"catalog":[470],"guideline\"":[472],"established":[475],"way.":[477],"Our":[478],"activities":[479],"might":[480],"help":[482],"teams":[486],"prepare":[488],"themselves":[489],"future,":[491],"legally":[492],"required":[493],"audits":[494],"their":[496],"newly":[497],"created":[498],"algorithms/AI":[500],"products.":[501]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}