{"id":"https://openalex.org/W4368232695","doi":"https://doi.org/10.1109/tai.2023.3272876","title":"An Explainable and Actionable Mistrust Scoring Framework for Model Monitoring","display_name":"An Explainable and Actionable Mistrust Scoring Framework for Model Monitoring","publication_year":2023,"publication_date":"2023-05-03","ids":{"openalex":"https://openalex.org/W4368232695","doi":"https://doi.org/10.1109/tai.2023.3272876"},"language":"en","primary_location":{"id":"doi:10.1109/tai.2023.3272876","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2023.3272876","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058303017","display_name":"Nandita Bhaskhar","orcid":"https://orcid.org/0000-0003-3624-9169"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nandita Bhaskhar","raw_affiliation_strings":["Department of Electrical Engineering, Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3624-9169","affiliations":[{"raw_affiliation_string":"Department of Electrical Engineering, Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004965117","display_name":"Daniel L. Rubin","orcid":"https://orcid.org/0000-0001-5057-4369"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel L. Rubin","raw_affiliation_strings":["Departments of Biomedical Data Science, Radiology, and Medicine (Biomedical Informatics), Stanford University, Stanford, CA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5057-4369","affiliations":[{"raw_affiliation_string":"Departments of Biomedical Data Science, Radiology, and Medicine (Biomedical Informatics), Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077447259","display_name":"Christopher Lee\u2010Messer","orcid":"https://orcid.org/0000-0002-2938-6184"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher Lee-Messer","raw_affiliation_strings":["Department of Neurology, Stanford University, Stanford, CA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Neurology, Stanford University, Stanford, CA, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":1.5713,"has_fulltext":false,"cited_by_count":12,"citation_normalized_percentile":{"value":0.85764806,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"5","issue":"4","first_page":"1473","last_page":"1485"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7515302896499634},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4704745411872864},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4521137475967407},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.445991575717926},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43709269165992737},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.42844918370246887},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.42348015308380127},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.11595174670219421}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7515302896499634},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4704745411872864},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4521137475967407},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.445991575717926},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43709269165992737},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.42844918370246887},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.42348015308380127},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.11595174670219421},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tai.2023.3272876","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2023.3272876","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":91,"referenced_works":["https://openalex.org/W1567512734","https://openalex.org/W1849277567","https://openalex.org/W1932198206","https://openalex.org/W2001610032","https://openalex.org/W2028138594","https://openalex.org/W2099419573","https://openalex.org/W2105497548","https://openalex.org/W2105934661","https://openalex.org/W2112796928","https://openalex.org/W2120587290","https://openalex.org/W2144182447","https://openalex.org/W2194775991","https://openalex.org/W2296719434","https://openalex.org/W2345279893","https://openalex.org/W2462906003","https://openalex.org/W2515822248","https://openalex.org/W2516809705","https://openalex.org/W2529337537","https://openalex.org/W2552396116","https://openalex.org/W2599354622","https://openalex.org/W2734358244","https://openalex.org/W2750384547","https://openalex.org/W2786599352","https://openalex.org/W2786712888","https://openalex.org/W2793165286","https://openalex.org/W2797583228","https://openalex.org/W2889326414","https://openalex.org/W2902986194","https://openalex.org/W2905813690","https://openalex.org/W2913300775","https://openalex.org/W2944016032","https://openalex.org/W2962790223","https://openalex.org/W2962851944","https://openalex.org/W2963049059","https://openalex.org/W2963290013","https://openalex.org/W2976969310","https://openalex.org/W2997167991","https://openalex.org/W2997212544","https://openalex.org/W3016911444","https://openalex.org/W3085380432","https://openalex.org/W3089860116","https://openalex.org/W3118608800","https://openalex.org/W3206874785","https://openalex.org/W4225727515","https://openalex.org/W4240490791","https://openalex.org/W4287724856","https://openalex.org/W6633808362","https://openalex.org/W6640425456","https://openalex.org/W6678914141","https://openalex.org/W6684191040","https://openalex.org/W6684809622","https://openalex.org/W6685133223","https://openalex.org/W6718836005","https://openalex.org/W6728622933","https://openalex.org/W6729822694","https://openalex.org/W6730042731","https://openalex.org/W6732209190","https://openalex.org/W6734194636","https://openalex.org/W6736518430","https://openalex.org/W6737947904","https://openalex.org/W6739651123","https://openalex.org/W6743688258","https://openalex.org/W6745891213","https://openalex.org/W6748102297","https://openalex.org/W6748163547","https://openalex.org/W6748381273","https://openalex.org/W6748865747","https://openalex.org/W6749162425","https://openalex.org/W6750665317","https://openalex.org/W6751800415","https://openalex.org/W6752760542","https://openalex.org/W6753123669","https://openalex.org/W6754011391","https://openalex.org/W6755655770","https://openalex.org/W6755672337","https://openalex.org/W6757574792","https://openalex.org/W6757615711","https://openalex.org/W6758976440","https://openalex.org/W6762924995","https://openalex.org/W6763324549","https://openalex.org/W6764214684","https://openalex.org/W6765696844","https://openalex.org/W6767996718","https://openalex.org/W6771731465","https://openalex.org/W6774683452","https://openalex.org/W6780616643","https://openalex.org/W6780874654","https://openalex.org/W6783213970","https://openalex.org/W6784323503","https://openalex.org/W6787972765","https://openalex.org/W6789731502"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Continuous":[0],"monitoring":[1],"of":[2,49,58,95,158,208],"trained":[3],"ML":[4],"models":[5],"to":[6,27,183],"determine":[7],"when":[8],"their":[9,20],"predictions":[10],"should":[11,13],"and":[12,32,74,82,120,130,135,163,219,231],"not":[14],"be":[15,28],"trusted":[16],"is":[17],"essential":[18],"for":[19,41,104,146,190,213,234],"safe":[21],"deployment.":[22],"Such":[23],"a":[24,37,56,100],"framework":[25,40],"ought":[26],"high-performing,":[29],"explainable,post":[30],"hoc,":[31],"actionable.":[33],"We":[34,45,108,125,173,194],"propose":[35],"TRUST-LAPSE,":[36],"\u201cmistrust\u201d":[38],"scoring":[39],"continuous":[42,106],"model":[43,53,192],"monitoring.":[44,107,193],"assess":[46],"the":[47,80,93,209],"trustworthiness":[48],"each":[50],"input":[51,97,118,184],"sample's":[52],"prediction":[54],"using":[55,69,132],"sequence":[57,94],"latent-space":[59,64,150],"embeddings.":[60],"Specifically,":[61],"1)":[62,115],"our":[63,84,138,197,225],"mistrust":[65,68,86,151,199,226],"score":[66,87],"estimates":[67],"distance":[70],"metrics":[71,76],"(Mahalanobis":[72],"distance)":[73],"similarity":[75],"(cosine":[77],"similarity)":[78],"in":[79,90,99,177],"latent-space,":[81],"2)":[83,121],"sequential":[85,198],"determines":[88],"deviations":[89],"correlations":[91],"over":[92,170,206],"past":[96],"representations":[98],"nonparametric,":[101],"sliding-window-based":[102],"algorithm":[103],"actionable":[105],"evaluate":[109,126],"TRUST-LAPSE":[110],"via":[111],"two":[112],"downstream":[113],"tasks:":[114],"distributionally":[116],"shifted":[117],"detection;":[119],"data":[122],"drift":[123,203],"detection.":[124,148],"across":[127],"diverse":[128],"domains\u2014audio":[129],"vision":[131],"public":[133],"datasets":[134,145],"further":[136],"benchmark":[137],"approach":[139],"on":[140],"challenging,":[141],"real-world":[142,191],"electroencephalograms":[143],"(EEG)":[144],"seizure":[147],"Our":[149],"scores":[152,200,227],"achieve":[153,201],"state-of-the-art":[154],"results":[155],"with":[156],"AUROCs":[157],"84.1":[159],"(vision),":[160],"73.9":[161],"(audio),":[162],"77.1":[164],"(clinical":[165],"EEGs),":[166],"outperforming":[167],"baselines":[168,179],"by":[169],"10":[171],"points.":[172],"expose":[174],"critical":[175],"failures":[176],"popular":[178],"that":[180,196,224],"remain":[181],"insensitive":[182],"semantic":[185],"content,":[186],"rendering":[187],"them":[188],"unfit":[189],"show":[195,223],"high":[202],"detection":[204],"rates;":[205],"90%":[207],"streams":[210],"show$<\\!":[211],"20{\\%}$error":[212],"all":[214],"domains.":[215],"Through":[216],"extensive":[217],"qualitative":[218],"quantitative":[220],"evaluations,":[221],"we":[222],"are":[228],"more":[229],"robust":[230],"provide":[232],"explainability":[233],"easy":[235],"adoption":[236],"into":[237],"practice.":[238]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2023-05-05T00:00:00"}