{"id":"https://openalex.org/W4382238433","doi":"https://doi.org/10.3389/fcomp.2023.1114806","title":"Evaluating machine-generated explanations: a \u201cScorecard\u201d method for XAI measurement science","display_name":"Evaluating machine-generated explanations: a \u201cScorecard\u201d method for XAI measurement science","publication_year":2023,"publication_date":"2023-05-09","ids":{"openalex":"https://openalex.org/W4382238433","doi":"https://doi.org/10.3389/fcomp.2023.1114806"},"language":"en","primary_location":{"id":"doi:10.3389/fcomp.2023.1114806","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1114806","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1114806/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1114806/pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103000195","display_name":"Robert R. Hoffman","orcid":"https://orcid.org/0000-0002-1387-7659"},"institutions":[{"id":"https://openalex.org/I1335578998","display_name":"Florida Institute for Human and Machine Cognition","ror":"https://ror.org/02napvw46","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1335578998"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Robert R. Hoffman","raw_affiliation_strings":["Institute for Human and Machine Cognition, Pensacola, FL, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Human and Machine Cognition, Pensacola, FL, United States","institution_ids":["https://openalex.org/I1335578998"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061024010","display_name":"Mohammadreza Jalaeian","orcid":"https://orcid.org/0000-0002-2689-6007"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohammadreza Jalaeian","raw_affiliation_strings":["Department of Integrated Systems Engineering, Ohio State University, Columbus, OH, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Integrated Systems Engineering, Ohio State University, Columbus, OH, United States","institution_ids":["https://openalex.org/I52357470"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002270469","display_name":"Connor Tate","orcid":null},"institutions":[{"id":"https://openalex.org/I1335578998","display_name":"Florida Institute for Human and Machine Cognition","ror":"https://ror.org/02napvw46","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1335578998"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Connor Tate","raw_affiliation_strings":["Institute for Human and Machine Cognition, Pensacola, FL, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute for Human and Machine Cognition, Pensacola, FL, United States","institution_ids":["https://openalex.org/I1335578998"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088906165","display_name":"Gary Klein","orcid":"https://orcid.org/0000-0002-8430-3230"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gary Klein","raw_affiliation_strings":["MacroCognition, LLC, Dayton, OH, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MacroCognition, LLC, Dayton, OH, United States","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086667759","display_name":"Shane T. Mueller","orcid":"https://orcid.org/0000-0001-7418-4592"},"institutions":[{"id":"https://openalex.org/I11957088","display_name":"Michigan Technological University","ror":"https://ror.org/0036rpn28","country_code":"US","type":"education","lineage":["https://openalex.org/I11957088"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shane T. Mueller","raw_affiliation_strings":["Department of Cognitive and Learning Sciences, Michigan Technological University, Houghton, MI, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Cognitive and Learning Sciences, Michigan Technological University, Houghton, MI, United States","institution_ids":["https://openalex.org/I11957088"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5103000195"],"corresponding_institution_ids":["https://openalex.org/I1335578998"],"apc_list":{"value":1150,"currency":"USD","value_usd":1285},"apc_paid":{"value":1150,"currency":"USD","value_usd":1285},"fwci":1.9777,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.88645901,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"5","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9776999950408936,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.9498000144958496,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/balanced-scorecard","display_name":"Balanced scorecard","score":0.8277004957199097},{"id":"https://openalex.org/keywords/sensemaking","display_name":"Sensemaking","score":0.7891514897346497},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6329643726348877},{"id":"https://openalex.org/keywords/terminology","display_name":"Terminology","score":0.60822594165802},{"id":"https://openalex.org/keywords/knowledge-management","display_name":"Knowledge management","score":0.4624495804309845},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.42054662108421326},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.32112976908683777},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.2838476896286011},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.15131962299346924},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09042695164680481},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08646634221076965}],"concepts":[{"id":"https://openalex.org/C199004533","wikidata":"https://www.wikidata.org/wiki/Q210127","display_name":"Balanced scorecard","level":2,"score":0.8277004957199097},{"id":"https://openalex.org/C2780554381","wikidata":"https://www.wikidata.org/wiki/Q2063340","display_name":"Sensemaking","level":2,"score":0.7891514897346497},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6329643726348877},{"id":"https://openalex.org/C547195049","wikidata":"https://www.wikidata.org/wiki/Q1725664","display_name":"Terminology","level":2,"score":0.60822594165802},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.4624495804309845},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.42054662108421326},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.32112976908683777},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.2838476896286011},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.15131962299346924},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09042695164680481},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08646634221076965},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3389/fcomp.2023.1114806","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1114806","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1114806/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},{"id":"pmh:oai:digitalcommons.mtu.edu:michigantech-p-36426","is_oa":true,"landing_page_url":"https://digitalcommons.mtu.edu/michigantech-p/17127","pdf_url":null,"source":{"id":"https://openalex.org/S4377196391","display_name":"Digital Commons - Michigan Tech (Michigan Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I11957088","host_organization_name":"Michigan Technological University","host_organization_lineage":["https://openalex.org/I11957088"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Michigan Tech Publications, Part 1","raw_type":"text"},{"id":"pmh:oai:doaj.org/article:d0f1e68f2d66494da438531b08fa48dd","is_oa":true,"landing_page_url":"https://doaj.org/article/d0f1e68f2d66494da438531b08fa48dd","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Frontiers in Computer Science, Vol 5 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3389/fcomp.2023.1114806","is_oa":true,"landing_page_url":"https://doi.org/10.3389/fcomp.2023.1114806","pdf_url":"https://www.frontiersin.org/articles/10.3389/fcomp.2023.1114806/pdf","source":{"id":"https://openalex.org/S4210211086","display_name":"Frontiers in Computer Science","issn_l":"2624-9898","issn":["2624-9898"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320527","host_organization_name":"Frontiers Media","host_organization_lineage":["https://openalex.org/P4310320527"],"host_organization_lineage_names":["Frontiers Media"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Computer Science","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.6800000071525574,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320311652","display_name":"Michigan Technological University","ror":"https://ror.org/0036rpn28"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320332815","display_name":"Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4382238433.pdf"},"referenced_works_count":88,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1518396806","https://openalex.org/W1607675442","https://openalex.org/W1990451873","https://openalex.org/W2018383914","https://openalex.org/W2045385178","https://openalex.org/W2057229779","https://openalex.org/W2070841423","https://openalex.org/W2074801235","https://openalex.org/W2088344354","https://openalex.org/W2101085453","https://openalex.org/W2101834258","https://openalex.org/W2116817443","https://openalex.org/W2125967820","https://openalex.org/W2129645506","https://openalex.org/W2130677218","https://openalex.org/W2157533992","https://openalex.org/W2338349451","https://openalex.org/W2461606751","https://openalex.org/W2511953903","https://openalex.org/W2513866468","https://openalex.org/W2556340658","https://openalex.org/W2587299461","https://openalex.org/W2788362053","https://openalex.org/W2788830130","https://openalex.org/W2806598546","https://openalex.org/W2807614947","https://openalex.org/W2891503716","https://openalex.org/W2894566090","https://openalex.org/W2896074035","https://openalex.org/W2896646377","https://openalex.org/W2909392392","https://openalex.org/W2913693384","https://openalex.org/W2941978148","https://openalex.org/W2942444880","https://openalex.org/W2958089299","https://openalex.org/W2960259221","https://openalex.org/W2962772482","https://openalex.org/W2963125461","https://openalex.org/W2963395533","https://openalex.org/W2963420481","https://openalex.org/W2963707011","https://openalex.org/W2964772144","https://openalex.org/W2966578325","https://openalex.org/W2992247793","https://openalex.org/W2995991797","https://openalex.org/W2997071275","https://openalex.org/W2999765337","https://openalex.org/W3000965188","https://openalex.org/W3001062618","https://openalex.org/W3015449728","https://openalex.org/W3098384022","https://openalex.org/W3104847483","https://openalex.org/W3107856351","https://openalex.org/W3118732602","https://openalex.org/W3134080822","https://openalex.org/W3174752098","https://openalex.org/W3197347140","https://openalex.org/W3199732472","https://openalex.org/W3204549625","https://openalex.org/W3211099476","https://openalex.org/W3216029730","https://openalex.org/W4200331401","https://openalex.org/W4206276140","https://openalex.org/W4220961556","https://openalex.org/W4225993386","https://openalex.org/W4230172260","https://openalex.org/W4231827019","https://openalex.org/W4236470361","https://openalex.org/W4243541132","https://openalex.org/W4253111388","https://openalex.org/W4288414189","https://openalex.org/W4289302522","https://openalex.org/W4293153764","https://openalex.org/W4293704399","https://openalex.org/W4296982335","https://openalex.org/W4298135002","https://openalex.org/W4319321305","https://openalex.org/W6735632633","https://openalex.org/W6737947904","https://openalex.org/W6752261457","https://openalex.org/W6756229586","https://openalex.org/W6758979359","https://openalex.org/W6761184903","https://openalex.org/W6762929394","https://openalex.org/W6764821371","https://openalex.org/W6770935282","https://openalex.org/W6772621168"],"related_works":["https://openalex.org/W3002559787","https://openalex.org/W2100609754","https://openalex.org/W2050640900","https://openalex.org/W2049050102","https://openalex.org/W2596767525","https://openalex.org/W1886987011","https://openalex.org/W2795557596","https://openalex.org/W3045759591","https://openalex.org/W2023412717","https://openalex.org/W2901081896"],"abstract_inverted_index":{"Introduction":[0],"Many":[1],"Explainable":[2],"AI":[3],"(XAI)":[4],"systems":[5,65,87,159,203,226],"provide":[6,74],"explanations":[7,128,179],"that":[8,46,122,157,189],"are":[9,227],"just":[10],"clues":[11],"or":[12,24],"hints":[13],"about":[14],"the":[15,69,76,92,124,130,140,182,210,216,222],"computational":[16],"models-Such":[17],"things":[18,52],"as":[19,37,89],"feature":[20],"lists,":[21],"decision":[22],"trees,":[23],"saliency":[25],"images.":[26],"However,":[27],"a":[28,98,114,186],"user":[29,183],"might":[30,58,80,96,175],"want":[31],"answers":[32],"to":[33,68,90,102,126,171,180,209],"deeper":[34],"questions":[35],"such":[36],"How":[38,57,79],"does":[39],"it":[40,44,54],"work?,":[41],"Why":[42],"did":[43],"do":[45],"instead":[47],"of":[48,71,120,136,153,224],"something":[49],"else?":[50],"What":[51,95],"can":[53,166,204],"get":[55],"wrong?":[56],"XAI":[59,64,81,86,202,225],"system":[60,82],"developers":[61,83,104,170],"evaluate":[62],"existing":[63],"with":[66,198,207],"regard":[67,208],"depth":[70,119],"support":[72,91,129,181],"they":[73,174],"for":[75,200],"user's":[77,93,131],"sensemaking?":[78,94],"shape":[84],"new":[85],"so":[88],"be":[97,167,205],"useful":[99],"conceptual":[100],"terminology":[101],"assist":[103],"in":[105,150,184,218],"approaching":[106],"this":[107,137],"challenge?":[108],"Method":[109],"Based":[110],"on":[111,221],"cognitive":[112,211],"theory,":[113],"scale":[115,138],"was":[116,148],"developed":[117],"reflecting":[118],"explanation,":[121],"is,":[123],"degree":[125],"which":[127,219],"sensemaking.":[132],"The":[133,146,164,195],"seven":[134],"levels":[135],"form":[139],"Explanation":[141],"Scorecard.":[142],"Results":[143],"and":[144,193,213],"discussion":[145],"Scorecard":[147,165],"utilized":[149],"an":[151],"analysis":[152],"recent":[154],"literature,":[155],"showing":[156],"many":[158],"still":[160],"present":[161],"low-level":[162],"explanations.":[163],"used":[168],"by":[169],"conceptualize":[172],"how":[173,201],"extend":[176],"their":[177],"machine-generated":[178],"developing":[185],"mental":[187],"model":[188],"instills":[190],"appropriate":[191],"trust":[192],"reliance.":[194],"article":[196],"concludes":[197],"recommendations":[199,214],"improved":[206],"considerations,":[212],"regarding":[215],"manner":[217],"results":[220],"evaluation":[223],"reported.":[228]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}