{"id":"https://openalex.org/W4328007192","doi":"https://doi.org/10.1109/tvcg.2023.3259341","title":"Are Metrics Enough? Guidelines for Communicating and Visualizing Predictive Models to Subject Matter Experts","display_name":"Are Metrics Enough? Guidelines for Communicating and Visualizing Predictive Models to Subject Matter Experts","publication_year":2023,"publication_date":"2023-03-20","ids":{"openalex":"https://openalex.org/W4328007192","doi":"https://doi.org/10.1109/tvcg.2023.3259341","pmid":"https://pubmed.ncbi.nlm.nih.gov/37030764"},"language":"en","primary_location":{"id":"doi:10.1109/tvcg.2023.3259341","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2023.3259341","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084507075","display_name":"Ashley Suh","orcid":"https://orcid.org/0000-0001-6513-8447"},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ashley Suh","raw_affiliation_strings":["Tufts University, Medford, MA, USA"],"raw_orcid":"https://orcid.org/0000-0001-6513-8447","affiliations":[{"raw_affiliation_string":"Tufts University, Medford, MA, USA","institution_ids":["https://openalex.org/I121934306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078304037","display_name":"Gabriel Appleby","orcid":"https://orcid.org/0000-0003-2436-2121"},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gabriel Appleby","raw_affiliation_strings":["Tufts University, Medford, MA, USA"],"raw_orcid":"https://orcid.org/0000-0003-2436-2121","affiliations":[{"raw_affiliation_string":"Tufts University, Medford, MA, USA","institution_ids":["https://openalex.org/I121934306"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005510639","display_name":"Erik Anderson","orcid":"https://orcid.org/0000-0002-0334-8497"},"institutions":[{"id":"https://openalex.org/I1283582996","display_name":"Novartis (Switzerland)","ror":"https://ror.org/02f9zrr09","country_code":"CH","type":"company","lineage":["https://openalex.org/I1283582996"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Erik W. Anderson","raw_affiliation_strings":["Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-0334-8497","affiliations":[{"raw_affiliation_string":"Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland","institution_ids":["https://openalex.org/I1283582996"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017390856","display_name":"Luca A. Finelli","orcid":null},"institutions":[{"id":"https://openalex.org/I1283582996","display_name":"Novartis (Switzerland)","ror":"https://ror.org/02f9zrr09","country_code":"CH","type":"company","lineage":["https://openalex.org/I1283582996"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Luca Finelli","raw_affiliation_strings":["Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland","institution_ids":["https://openalex.org/I1283582996"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089451178","display_name":"Remco Chang","orcid":"https://orcid.org/0000-0002-6484-6430"},"institutions":[{"id":"https://openalex.org/I121934306","display_name":"Tufts University","ror":"https://ror.org/05wvpxv85","country_code":"US","type":"education","lineage":["https://openalex.org/I121934306"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Remco Chang","raw_affiliation_strings":["Tufts University, Medford, MA, USA"],"raw_orcid":"https://orcid.org/0000-0002-6484-6430","affiliations":[{"raw_affiliation_string":"Tufts University, Medford, MA, USA","institution_ids":["https://openalex.org/I121934306"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033915638","display_name":"Dylan Cashman","orcid":"https://orcid.org/0000-0003-4853-5701"},"institutions":[{"id":"https://openalex.org/I1283582996","display_name":"Novartis (Switzerland)","ror":"https://ror.org/02f9zrr09","country_code":"CH","type":"company","lineage":["https://openalex.org/I1283582996"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Dylan Cashman","raw_affiliation_strings":["Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland"],"raw_orcid":"https://orcid.org/0000-0003-4853-5701","affiliations":[{"raw_affiliation_string":"Novartis Pharmaceuticals Corporation, Data Science and AI, Basel, Switzerland","institution_ids":["https://openalex.org/I1283582996"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5084507075"],"corresponding_institution_ids":["https://openalex.org/I121934306"],"apc_list":null,"apc_paid":null,"fwci":1.5337,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.85627529,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"30","issue":"7","first_page":"4137","last_page":"4153"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.9921000003814697,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9884999990463257,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7557187080383301},{"id":"https://openalex.org/keywords/subject-matter","display_name":"Subject matter","score":0.5854030251502991},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.5264150500297546},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5109740495681763},{"id":"https://openalex.org/keywords/data-visualization","display_name":"Data visualization","score":0.4946558177471161},{"id":"https://openalex.org/keywords/subject-matter-expert","display_name":"Subject-matter expert","score":0.4496572017669678},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3872451186180115},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3319249749183655},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.33070212602615356}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7557187080383301},{"id":"https://openalex.org/C2992860105","wikidata":"https://www.wikidata.org/wiki/Q7631392","display_name":"Subject matter","level":3,"score":0.5854030251502991},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.5264150500297546},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5109740495681763},{"id":"https://openalex.org/C172367668","wikidata":"https://www.wikidata.org/wiki/Q6504956","display_name":"Data visualization","level":3,"score":0.4946558177471161},{"id":"https://openalex.org/C105002631","wikidata":"https://www.wikidata.org/wiki/Q4833645","display_name":"Subject-matter expert","level":3,"score":0.4496572017669678},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3872451186180115},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3319249749183655},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33070212602615356},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.0},{"id":"https://openalex.org/C47177190","wikidata":"https://www.wikidata.org/wiki/Q207137","display_name":"Curriculum","level":2,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D003142","descriptor_name":"Communication","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003142","descriptor_name":"Communication","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003142","descriptor_name":"Communication","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003142","descriptor_name":"Communication","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D003196","descriptor_name":"Computer Graphics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003196","descriptor_name":"Computer Graphics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003196","descriptor_name":"Computer Graphics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D003196","descriptor_name":"Computer Graphics","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D006801","descriptor_name":"Humans","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D017408","descriptor_name":"Guidelines as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D017408","descriptor_name":"Guidelines as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D017408","descriptor_name":"Guidelines as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D017408","descriptor_name":"Guidelines as Topic","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1109/tvcg.2023.3259341","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tvcg.2023.3259341","pdf_url":null,"source":{"id":"https://openalex.org/S84775595","display_name":"IEEE Transactions on Visualization and Computer Graphics","issn_l":"1077-2626","issn":["1077-2626","1941-0506","2160-9306"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Visualization and Computer Graphics","raw_type":"journal-article"},{"id":"pmid:37030764","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/37030764","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on visualization and computer graphics","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1425119500","display_name":null,"funder_award_id":"OAC-1939945","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3475743935","display_name":null,"funder_award_id":"OAC-1940175","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G822117293","display_name":null,"funder_award_id":"IIS1452977","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8492696410","display_name":null,"funder_award_id":"OAC-2118201","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8513907812","display_name":null,"funder_award_id":"NRT-2021874","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":61,"referenced_works":["https://openalex.org/W643796550","https://openalex.org/W1967320885","https://openalex.org/W2016874080","https://openalex.org/W2021257274","https://openalex.org/W2023029656","https://openalex.org/W2023320048","https://openalex.org/W2044102377","https://openalex.org/W2050733067","https://openalex.org/W2089331794","https://openalex.org/W2091525666","https://openalex.org/W2113386367","https://openalex.org/W2127561630","https://openalex.org/W2157900633","https://openalex.org/W2163805179","https://openalex.org/W2169628183","https://openalex.org/W2295598076","https://openalex.org/W2594475271","https://openalex.org/W2664267452","https://openalex.org/W2790102940","https://openalex.org/W2806375370","https://openalex.org/W2896774728","https://openalex.org/W2902883805","https://openalex.org/W2904955348","https://openalex.org/W2913589238","https://openalex.org/W2923421605","https://openalex.org/W2942444880","https://openalex.org/W2946711149","https://openalex.org/W2963355221","https://openalex.org/W2963795072","https://openalex.org/W2967844572","https://openalex.org/W2991157178","https://openalex.org/W2994958466","https://openalex.org/W2995991797","https://openalex.org/W3000686922","https://openalex.org/W3009578469","https://openalex.org/W3019489177","https://openalex.org/W3034069517","https://openalex.org/W3048742909","https://openalex.org/W3094507803","https://openalex.org/W3100758187","https://openalex.org/W3115483167","https://openalex.org/W3118813946","https://openalex.org/W3122778363","https://openalex.org/W3123497755","https://openalex.org/W3130185945","https://openalex.org/W3137125108","https://openalex.org/W3138819813","https://openalex.org/W3160037564","https://openalex.org/W3190429087","https://openalex.org/W3203348860","https://openalex.org/W4221015417","https://openalex.org/W4230130496","https://openalex.org/W4288083705","https://openalex.org/W4288286466","https://openalex.org/W4289123308","https://openalex.org/W4318697169","https://openalex.org/W6734862562","https://openalex.org/W6756943755","https://openalex.org/W6757275317","https://openalex.org/W6765879183","https://openalex.org/W6766586947"],"related_works":["https://openalex.org/W3206014578","https://openalex.org/W2188803587","https://openalex.org/W2013728941","https://openalex.org/W4225274103","https://openalex.org/W2154046714","https://openalex.org/W2189613078","https://openalex.org/W2579659702","https://openalex.org/W2923661510","https://openalex.org/W1574055964","https://openalex.org/W1965329638"],"abstract_inverted_index":{"Presenting":[0],"a":[1,6,31,55,58,163,172,182,186,192,216],"predictive":[2],"model's":[3,217,234,242],"performance":[4],"is":[5],"communication":[7,108,166],"bottleneck":[8],"that":[9,168],"threatens":[10],"collaborations":[11],"between":[12,109],"data":[13,101,126],"scientists":[14,102],"and":[15,20,37,100,127,139,149,179,196,228,239,246],"subject":[16,44,71,97,203,209],"matter":[17,45,72,98,204,210],"experts.":[18,205],"Accuracy":[19],"error":[21],"metrics":[22],"alone":[23],"fail":[24,62],"to":[25,47,53,103,147,231],"tell":[26],"the":[27,105,117,125,130,143,177,222,225,233,241],"whole":[28],"story":[29],"of":[30,80,123,129,145,165,181,188,221],"model":[32],"-":[33,39,141,236],"its":[34],"risks,":[35],"strengths,":[36],"limitations":[38],"making":[40],"it":[41],"difficult":[42],"for":[43,175,224],"experts":[46,73,99,211],"feel":[48],"confident":[49],"in":[50,63,78,107,191],"their":[51,200],"decision":[52],"use":[54,169,201,243],"model.":[56,183],"As":[57],"result,":[59],"models":[60,77],"may":[61],"unexpected":[64],"ways":[65],"or":[66],"go":[67],"entirely":[68],"unused,":[69],"as":[70,171],"disregard":[74],"poorly":[75],"presented":[76,226],"favor":[79],"familiar,":[81],"yet":[82],"arguably":[83],"substandard":[84],"methods.":[85],"In":[86],"this":[87],"paper,":[88],"we":[89,161],"describe":[90],"an":[91],"iterative":[92],"study":[93],"conducted":[94],"with":[95],"both":[96],"understand":[104],"gaps":[106],"these":[110],"two":[111,118],"groups.":[112],"We":[113,184],"find":[114],"that,":[115],"while":[116],"groups":[119],"share":[120],"common":[121,173],"goals":[122],"understanding":[124],"predictions":[128],"model,":[131,227],"friction":[132],"can":[133],"stem":[134],"from":[135,202],"unfamiliar":[136],"terms,":[137],"metrics,":[138],"visualizations":[140],"limiting":[142],"transfer":[144],"knowledge":[146],"SMEs":[148],"discouraging":[150],"clarifying":[151],"questions":[152],"being":[153],"asked":[154],"during":[155],"presentations.":[156],"Based":[157],"on":[158,199],"our":[159,189,207],"findings,":[160],"derive":[162],"set":[164],"guidelines":[167,190],"visualization":[170],"medium":[174],"communicating":[176],"strengths":[178],"weaknesses":[180],"provide":[185],"demonstration":[187],"regression":[193],"modeling":[194],"scenario":[195],"elicit":[197],"feedback":[198],"From":[206],"demonstration,":[208],"were":[212],"more":[213,219],"comfortable":[214],"discussing":[215],"performance,":[218],"aware":[220],"trade-offs":[223],"better":[229],"equipped":[230],"assess":[232],"risks":[235],"ultimately":[237],"informing":[238],"contextualizing":[240],"beyond":[244],"text":[245],"numbers.":[247]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}