{"id":"https://openalex.org/W3135989381","doi":"https://doi.org/10.1145/3450439.3451864","title":"Learning to safely approve updates to machine learning algorithms","display_name":"Learning to safely approve updates to machine learning algorithms","publication_year":2021,"publication_date":"2021-03-23","ids":{"openalex":"https://openalex.org/W3135989381","doi":"https://doi.org/10.1145/3450439.3451864","mag":"3135989381"},"language":"en","primary_location":{"id":"doi:10.1145/3450439.3451864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3450439.3451864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3450439.3451864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Conference on Health, Inference, and Learning","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3450439.3451864","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051492770","display_name":"Jean Feng","orcid":"https://orcid.org/0000-0003-2041-3104"},"institutions":[{"id":"https://openalex.org/I2803209242","display_name":"University of California System","ror":"https://ror.org/00pjdza24","country_code":"US","type":"education","lineage":["https://openalex.org/I2803209242"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Jean Feng","raw_affiliation_strings":["University of California"],"affiliations":[{"raw_affiliation_string":"University of California","institution_ids":["https://openalex.org/I2803209242"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5051492770"],"corresponding_institution_ids":["https://openalex.org/I2803209242"],"apc_list":null,"apc_paid":null,"fwci":1.0741,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.83647565,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"164","last_page":"173"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11396","display_name":"Artificial Intelligence in Healthcare","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/3605","display_name":"Health Information Management"},"field":{"id":"https://openalex.org/fields/36","display_name":"Health Professions"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T11775","display_name":"COVID-19 diagnosis using AI","score":0.994700014591217,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7366635799407959},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5239667892456055},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4597179889678955},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4176720380783081}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7366635799407959},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5239667892456055},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4597179889678955},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4176720380783081}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3450439.3451864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3450439.3451864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3450439.3451864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Conference on Health, Inference, and Learning","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3450439.3451864","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3450439.3451864","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3450439.3451864","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Conference on Health, Inference, and Learning","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1814800902","display_name":null,"funder_award_id":"U01FD005978","funder_id":"https://openalex.org/F4320332744","funder_display_name":"Center of Excellence in Regulatory Science and Innovation"},{"id":"https://openalex.org/G4345674621","display_name":null,"funder_award_id":"U01FD005978","funder_id":"https://openalex.org/F4320306085","funder_display_name":"U.S. Department of Health and Human Services"},{"id":"https://openalex.org/G7524809423","display_name":null,"funder_award_id":"(HHS)","funder_id":"https://openalex.org/F4320306085","funder_display_name":"U.S. Department of Health and Human Services"}],"funders":[{"id":"https://openalex.org/F4320306085","display_name":"U.S. Department of Health and Human Services","ror":"https://ror.org/033jnv181"},{"id":"https://openalex.org/F4320332622","display_name":"University of California, San Francisco","ror":"https://ror.org/043mz5j54"},{"id":"https://openalex.org/F4320332744","display_name":"Center of Excellence in Regulatory Science and Innovation","ror":"https://ror.org/05vzafd60"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3135989381.pdf","grobid_xml":"https://content.openalex.org/works/W3135989381.grobid-xml"},"referenced_works_count":32,"referenced_works":["https://openalex.org/W1593784838","https://openalex.org/W1729157025","https://openalex.org/W1878322007","https://openalex.org/W2096846143","https://openalex.org/W2101946573","https://openalex.org/W2113892508","https://openalex.org/W2138162199","https://openalex.org/W2141102048","https://openalex.org/W2146297376","https://openalex.org/W2149107760","https://openalex.org/W2161813894","https://openalex.org/W2171809276","https://openalex.org/W2186453173","https://openalex.org/W2212660284","https://openalex.org/W2317648909","https://openalex.org/W2557592025","https://openalex.org/W2597505554","https://openalex.org/W2604834158","https://openalex.org/W2753858529","https://openalex.org/W2952786955","https://openalex.org/W2963818033","https://openalex.org/W2965485927","https://openalex.org/W2967329333","https://openalex.org/W2981869278","https://openalex.org/W2982580298","https://openalex.org/W2991319174","https://openalex.org/W3006990238","https://openalex.org/W3024029620","https://openalex.org/W3031989616","https://openalex.org/W3081126843","https://openalex.org/W3089324918","https://openalex.org/W3101973032"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347","https://openalex.org/W4210805261"],"abstract_inverted_index":{"Machine":[0],"learning":[1,124],"algorithms":[2,85],"in":[3,71,86,145],"healthcare":[4,16],"have":[5,31],"the":[6,28,62,87,97,125,161,177,185,195],"potential":[7],"to":[8,20,36,40,83,183,220],"continually":[9],"learn":[10,184],"from":[11],"real-world":[12],"data":[13,63],"generated":[14],"during":[15],"delivery":[17],"and":[18,50,104,189,203],"adapt":[19],"dataset":[21],"shifts.":[22,91],"As":[23],"such,":[24],"regulatory":[25],"bodies":[26],"like":[27],"US":[29],"FDA":[30],"begun":[32],"discussions":[33],"on":[34],"how":[35],"autonomously":[37],"approve":[38],"modifications":[39,46,82,107],"algorithms.":[41],"Current":[42],"proposals":[43],"evaluate":[44],"algorithmic":[45],"via":[47],"hypothesis":[48],"testing":[49],"control":[51],"a":[52,116,120,132,139,169,215],"definition":[53],"of":[54,89,134,141,148,160,230],"online":[55],"approval":[56,79,98,118,127,135,210,229],"error":[57],"that":[58,96,143,172,208],"only":[59],"applies":[60],"if":[61],"is":[64,69,95,214],"stationary":[65],"over":[66,131],"time,":[67],"which":[68],"unlikely":[70],"practice.":[72],"To":[73,153],"this":[74,166],"end,":[75],"we":[76,122,206],"investigate":[77],"designing":[78],"policies":[80],"for":[81],"ML":[84,162],"presence":[88],"distributional":[90,196,223],"Our":[92],"key":[93],"observation":[94],"policy":[99,119,128],"most":[100,186],"efficient":[101],"at":[102],"identifying":[103],"approving":[105,151],"beneficial":[106,231],"varies":[108],"across":[109],"problem":[110],"settings.":[111],"So,":[112],"rather":[113],"than":[114],"selecting":[115],"fixed":[117],"priori,":[121],"propose":[123],"best":[126],"by":[129],"searching":[130],"family":[133,140,167],"strategies.":[136],"We":[137,175],"define":[138],"strategies":[142,211],"range":[144],"their":[146],"level":[147],"optimism":[149],"when":[150],"modifications.":[152,232],"protect":[154,221],"against":[155,222],"settings":[156],"where":[157],"no":[158],"version":[159],"algorithm":[163],"performs":[164],"well,":[165],"includes":[168],"pessimistic":[170],"strategy":[171,188],"rescinds":[173],"approval.":[174],"use":[176],"exponentially":[178],"weighted":[179],"averaging":[180],"forecaster":[181],"(EWAF)":[182],"appropriate":[187],"derive":[190],"tighter":[191],"regret":[192],"bounds":[193],"assuming":[194],"shifts":[197,224],"are":[198],"bounded.":[199],"In":[200],"simulation":[201],"studies":[202],"empirical":[204],"analyses,":[205],"find":[207],"wrapping":[209],"within":[212],"EWAF":[213],"simple":[216],"yet":[217],"effective":[218],"approach":[219],"without":[225],"significantly":[226],"slowing":[227],"down":[228]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}