{"id":"https://openalex.org/W4387571658","doi":"https://doi.org/10.3390/e25101443","title":"Closed-Loop Uncertainty: The Evaluation and Calibration of Uncertainty for Human\u2013Machine Teams under Data Drift","display_name":"Closed-Loop Uncertainty: The Evaluation and Calibration of Uncertainty for Human\u2013Machine Teams under Data Drift","publication_year":2023,"publication_date":"2023-10-12","ids":{"openalex":"https://openalex.org/W4387571658","doi":"https://doi.org/10.3390/e25101443","pmid":"https://pubmed.ncbi.nlm.nih.gov/37895564"},"language":"en","primary_location":{"id":"doi:10.3390/e25101443","is_oa":true,"landing_page_url":"http://dx.doi.org/10.3390/e25101443","pdf_url":"https://www.mdpi.com/1099-4300/25/10/1443/pdf?version=1697102201","source":{"id":"https://openalex.org/S195231649","display_name":"Entropy","issn_l":"1099-4300","issn":["1099-4300"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Entropy","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj","pubmed"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1099-4300/25/10/1443/pdf?version=1697102201","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093050025","display_name":"Zachary Bishof","orcid":null},"institutions":[{"id":"https://openalex.org/I2802384920","display_name":"Stennis Space Center","ror":"https://ror.org/03j9e2j92","country_code":"US","type":"facility","lineage":["https://openalex.org/I2802384920","https://openalex.org/I4210124779"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Zachary Bishof","raw_affiliation_strings":["U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA","institution_ids":["https://openalex.org/I2802384920"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054539260","display_name":"Jaelle Scheuerman","orcid":null},"institutions":[{"id":"https://openalex.org/I2802384920","display_name":"Stennis Space Center","ror":"https://ror.org/03j9e2j92","country_code":"US","type":"facility","lineage":["https://openalex.org/I2802384920","https://openalex.org/I4210124779"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaelle Scheuerman","raw_affiliation_strings":["U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA","institution_ids":["https://openalex.org/I2802384920"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040214460","display_name":"Chris J. Michael","orcid":null},"institutions":[{"id":"https://openalex.org/I2802384920","display_name":"Stennis Space Center","ror":"https://ror.org/03j9e2j92","country_code":"US","type":"facility","lineage":["https://openalex.org/I2802384920","https://openalex.org/I4210124779"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris J. Michael","raw_affiliation_strings":["U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"U.S. Naval Research Laboratory, 1005 Balch Boulevard, Stennis Space Center, St. Louis, MS 39529, USA","institution_ids":["https://openalex.org/I2802384920"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5093050025"],"corresponding_institution_ids":["https://openalex.org/I2802384920"],"apc_list":{"value":2000,"currency":"CHF","value_usd":2165},"apc_paid":{"value":2000,"currency":"CHF","value_usd":2165},"fwci":0.1685,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.57092474,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":94},"biblio":{"volume":"25","issue":"10","first_page":"1443","last_page":"1443"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9939000010490417,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.9624999761581421,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11443","display_name":"Advanced Statistical Process Monitoring","score":0.9509000182151794,"subfield":{"id":"https://openalex.org/subfields/1804","display_name":"Statistics, Probability and Uncertainty"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.8242093324661255},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7424003481864929},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.601252019405365},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5371208190917969},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5325385928153992},{"id":"https://openalex.org/keywords/uncertainty-reduction-theory","display_name":"Uncertainty reduction theory","score":0.5268144607543945},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.48291680216789246},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.44939523935317993},{"id":"https://openalex.org/keywords/measurement-uncertainty","display_name":"Measurement uncertainty","score":0.41577422618865967},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.36108267307281494},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14872002601623535},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.108366459608078}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.8242093324661255},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7424003481864929},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.601252019405365},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5371208190917969},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5325385928153992},{"id":"https://openalex.org/C94361409","wikidata":"https://www.wikidata.org/wiki/Q7882500","display_name":"Uncertainty reduction theory","level":2,"score":0.5268144607543945},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.48291680216789246},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.44939523935317993},{"id":"https://openalex.org/C137209882","wikidata":"https://www.wikidata.org/wiki/Q1403517","display_name":"Measurement uncertainty","level":2,"score":0.41577422618865967},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.36108267307281494},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14872002601623535},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.108366459608078},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.3390/e25101443","is_oa":true,"landing_page_url":"http://dx.doi.org/10.3390/e25101443","pdf_url":"https://www.mdpi.com/1099-4300/25/10/1443/pdf?version=1697102201","source":{"id":"https://openalex.org/S195231649","display_name":"Entropy","issn_l":"1099-4300","issn":["1099-4300"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Entropy","raw_type":"journal-article"},{"id":"pmid:37895564","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37895564","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Entropy (Basel, Switzerland)","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:10606420","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/10606420","pdf_url":"https://pmc.ncbi.nlm.nih.gov/articles/PMC10606420/pdf/entropy-25-01443.pdf","source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Entropy (Basel)","raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:2a1de2eab7ac461e9e6785079f02248e","is_oa":true,"landing_page_url":"https://doaj.org/article/2a1de2eab7ac461e9e6785079f02248e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Entropy, Vol 25, Iss 10, p 1443 (2023)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/1099-4300/25/10/1443/","is_oa":true,"landing_page_url":"https://doi.org/10.3390/e25101443","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Entropy","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/e25101443","is_oa":true,"landing_page_url":"http://dx.doi.org/10.3390/e25101443","pdf_url":"https://www.mdpi.com/1099-4300/25/10/1443/pdf?version=1697102201","source":{"id":"https://openalex.org/S195231649","display_name":"Entropy","issn_l":"1099-4300","issn":["1099-4300"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Entropy","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4387571658.pdf"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W349770100","https://openalex.org/W1524761913","https://openalex.org/W1548531598","https://openalex.org/W1554784324","https://openalex.org/W1567139212","https://openalex.org/W1977591411","https://openalex.org/W2003238113","https://openalex.org/W2037497826","https://openalex.org/W2069266605","https://openalex.org/W2083259944","https://openalex.org/W2111574971","https://openalex.org/W2169027522","https://openalex.org/W2245547611","https://openalex.org/W2620180909","https://openalex.org/W2739473244","https://openalex.org/W2887976372","https://openalex.org/W2888507760","https://openalex.org/W2899751989","https://openalex.org/W2945850646","https://openalex.org/W2981631836","https://openalex.org/W2986573009","https://openalex.org/W3014596384","https://openalex.org/W3102100346","https://openalex.org/W3134774296","https://openalex.org/W3164731060","https://openalex.org/W3183398589","https://openalex.org/W4214717370","https://openalex.org/W4241336979","https://openalex.org/W4292289324","https://openalex.org/W6611762666","https://openalex.org/W6631546530","https://openalex.org/W6739651123","https://openalex.org/W6751800415"],"related_works":["https://openalex.org/W3008339103","https://openalex.org/W2037453698","https://openalex.org/W4297093367","https://openalex.org/W1994350080","https://openalex.org/W4366834219","https://openalex.org/W3117144634","https://openalex.org/W2012386145","https://openalex.org/W4367857114","https://openalex.org/W1979953904","https://openalex.org/W2625650645"],"abstract_inverted_index":{"Though":[0],"an":[1,39,91],"accurate":[2],"measurement":[3],"of":[4,15,20,23,29,45,63,101,148,157,160,197,206],"entropy,":[5],"or":[6,60],"more":[7,149],"generally":[8],"uncertainty,":[9,218],"is":[10,32,135],"critical":[11],"to":[12,77,110,129,137,199],"the":[13,18,21,46,69,78,112,122,155,207,220],"success":[14],"human-machine":[16,50,143,208],"teams,":[17],"evaluation":[19],"accuracy":[22],"such":[24],"metrics":[25],"as":[26,38,98,111],"a":[27,86,99,106,118,126,130,164,171,179,194],"probability":[28,100],"machine":[30,72,102,161,185],"correctness":[31,162,186],"often":[33],"aggregated":[34],"and":[35,71,140],"not":[36,53],"assessed":[37],"iterative":[40],"control":[41],"process.":[42],"The":[43,176],"entropy":[44],"decisions":[47],"made":[48],"by":[49,89,183],"teams":[51],"may":[52,94],"be":[54,95,138],"accurately":[55],"measured":[56],"under":[57],"cold":[58],"start":[59],"at":[61,124,187,203],"times":[62],"data":[64],"drift":[65],"unless":[66],"disagreements":[67],"between":[68],"human":[70],"are":[73,191],"immediately":[74],"fed":[75],"back":[76],"classifier":[79],"iteratively.":[80],"In":[81],"this":[82],"study,":[83],"we":[84,153],"present":[85],"stochastic":[87],"framework":[88],"which":[90,116,125],"uncertainty":[92,168,181,202],"model":[93,169,182],"evaluated":[96],"iteratively":[97],"correctness.":[103],"We":[104],"target":[105],"novel":[107,172,214],"problem,":[108,115],"referred":[109],"threshold":[113],"selection":[114],"involves":[117],"user":[119],"subjectively":[120],"selecting":[121],"point":[123],"signal":[127],"transitions":[128],"low":[131],"state.":[132],"This":[133],"problem":[134],"designed":[136],"simple":[139],"replicable":[141],"for":[142],"experimentation":[144],"while":[145],"exhibiting":[146],"properties":[147],"complex":[150],"applications.":[151],"Finally,":[152],"explore":[154],"potential":[156],"incorporating":[158,184],"feedback":[159],"into":[163],"baseline":[165,180,221],"na\u00efve":[166],"Bayes":[167],"with":[170],"reinforcement":[173],"learning":[174],"approach.":[175],"approach":[177],"refines":[178],"every":[188,223],"iteration.":[189],"Experiments":[190],"conducted":[192],"over":[193],"large":[195],"number":[196],"realizations":[198],"properly":[200],"evaluate":[201],"each":[204],"iteration":[205],"team.":[209],"Results":[210],"show":[211],"that":[212],"our":[213],"approach,":[215],"called":[216],"closed-loop":[217],"outperforms":[219],"in":[222],"case,":[224],"yielding":[225],"about":[226],"45%":[227],"improvement":[228],"on":[229],"average.":[230]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}