{"id":"https://openalex.org/W2240067561","doi":"https://doi.org/10.1109/tnnls.2016.2599820","title":"Evaluating the Visualization of What a Deep Neural Network Has Learned","display_name":"Evaluating the Visualization of What a Deep Neural Network Has Learned","publication_year":2016,"publication_date":"2016-08-25","ids":{"openalex":"https://openalex.org/W2240067561","doi":"https://doi.org/10.1109/tnnls.2016.2599820","mag":"2240067561","pmid":"https://pubmed.ncbi.nlm.nih.gov/27576267"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2016.2599820","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2016.2599820","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5026451495","display_name":"Wojciech Samek","orcid":"https://orcid.org/0000-0002-6283-3265"},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Wojciech Samek","raw_affiliation_strings":["Fraunhofer Heinrich Hertz Institute, Berlin, Germany"],"raw_orcid":"https://orcid.org/0000-0002-6283-3265","affiliations":[{"raw_affiliation_string":"Fraunhofer Heinrich Hertz Institute, Berlin, Germany","institution_ids":["https://openalex.org/I2800274787"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072460783","display_name":"Alexander Binder","orcid":"https://orcid.org/0000-0001-9605-6209"},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Alexander Binder","raw_affiliation_strings":["Information Systems Technology and Design Pillar, Singapore University of Technology and Design, Singapore"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Information Systems Technology and Design Pillar, Singapore University of Technology and Design, Singapore","institution_ids":["https://openalex.org/I152815399"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026240645","display_name":"Gr\u00e9goire Montavon","orcid":"https://orcid.org/0000-0001-7243-6186"},"institutions":[{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Gregoire Montavon","raw_affiliation_strings":["Technische Universit\u00e4t Berlin, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017608155","display_name":"Sebastian Lapuschkin","orcid":"https://orcid.org/0000-0002-0762-7258"},"institutions":[{"id":"https://openalex.org/I2800274787","display_name":"Fraunhofer Institute for Telecommunications, Heinrich Hertz Institute","ror":"https://ror.org/02tbr6331","country_code":"DE","type":"facility","lineage":["https://openalex.org/I2800274787","https://openalex.org/I4923324"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Sebastian Lapuschkin","raw_affiliation_strings":["Fraunhofer Heinrich Hertz Institute, Berlin, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Fraunhofer Heinrich Hertz Institute, Berlin, Germany","institution_ids":["https://openalex.org/I2800274787"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107838719","display_name":"Klaus\u2010Robert M\u00fcller","orcid":null},"institutions":[{"id":"https://openalex.org/I197347611","display_name":"Korea University","ror":"https://ror.org/047dqcg40","country_code":"KR","type":"education","lineage":["https://openalex.org/I197347611"]},{"id":"https://openalex.org/I4577782","display_name":"Technische Universit\u00e4t Berlin","ror":"https://ror.org/03v4gjf40","country_code":"DE","type":"education","lineage":["https://openalex.org/I4577782"]}],"countries":["DE","KR"],"is_corresponding":false,"raw_author_name":"Klaus-Robert Muller","raw_affiliation_strings":["Technische Universit\u00e4t Berlin, Berlin, Germany","Department of Brain and Cognitive Engineering, Korea University, Seoul, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technische Universit\u00e4t Berlin, Berlin, Germany","institution_ids":["https://openalex.org/I4577782"]},{"raw_affiliation_string":"Department of Brain and Cognitive Engineering, Korea University, Seoul, South Korea","institution_ids":["https://openalex.org/I197347611"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":111.2547,"has_fulltext":false,"cited_by_count":1172,"citation_normalized_percentile":{"value":0.99953461,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"28","issue":"11","first_page":"2660","last_page":"2673"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.925599992275238,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.925599992275238,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10799","display_name":"Data Visualization and Analytics","score":0.013399999588727951,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.007799999788403511,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7655476927757263},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.6986721158027649},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6922028660774231},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.6441715359687805},{"id":"https://openalex.org/keywords/pixel","display_name":"Pixel","score":0.6110882759094238},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.5603482723236084},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5497936010360718},{"id":"https://openalex.org/keywords/grasp","display_name":"GRASP","score":0.43153998255729675}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7655476927757263},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.6986721158027649},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6922028660774231},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.6441715359687805},{"id":"https://openalex.org/C160633673","wikidata":"https://www.wikidata.org/wiki/Q355198","display_name":"Pixel","level":2,"score":0.6110882759094238},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.5603482723236084},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5497936010360718},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.43153998255729675},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/tnnls.2016.2599820","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2016.2599820","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:27576267","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/27576267","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null},{"id":"pmh:oai:fraunhofer.de:N-480389","is_oa":false,"landing_page_url":"http://publica.fraunhofer.de/documents/N-480389.html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400801","display_name":"Publikationsdatenbank der Fraunhofer-Gesellschaft (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Fraunhofer HHI","raw_type":"article"},{"id":"pmh:oai:publica.fraunhofer.de:publica/251789","is_oa":false,"landing_page_url":"https://publica.fraunhofer.de/handle/publica/251789","pdf_url":null,"source":{"id":"https://openalex.org/S4306400318","display_name":"Fraunhofer-Publica (Fraunhofer-Gesellschaft)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4923324","host_organization_name":"Fraunhofer-Gesellschaft","host_organization_lineage":["https://openalex.org/I4923324"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"journal article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7900000214576721,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G1281499274","display_name":null,"funder_award_id":"MU 987/17-1","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G1553489635","display_name":null,"funder_award_id":"01IS14013A.","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"},{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W237546731","https://openalex.org/W1493372406","https://openalex.org/W1498436455","https://openalex.org/W1576278180","https://openalex.org/W1673923490","https://openalex.org/W1731081199","https://openalex.org/W1787224781","https://openalex.org/W1825675169","https://openalex.org/W1849277567","https://openalex.org/W1899185266","https://openalex.org/W1915485278","https://openalex.org/W1932198206","https://openalex.org/W1945616565","https://openalex.org/W1996242427","https://openalex.org/W1999192586","https://openalex.org/W2017814585","https://openalex.org/W2033206800","https://openalex.org/W2047197335","https://openalex.org/W2054802006","https://openalex.org/W2061627538","https://openalex.org/W2097117768","https://openalex.org/W2117539524","https://openalex.org/W2134670479","https://openalex.org/W2141125852","https://openalex.org/W2144764737","https://openalex.org/W2150165932","https://openalex.org/W2151035455","https://openalex.org/W2158413235","https://openalex.org/W2158899491","https://openalex.org/W2163605009","https://openalex.org/W2167510172","https://openalex.org/W2195388612","https://openalex.org/W2251939518","https://openalex.org/W2403553945","https://openalex.org/W2521500513","https://openalex.org/W2914484425","https://openalex.org/W2949987032","https://openalex.org/W2950179405","https://openalex.org/W2952230511","https://openalex.org/W2962851944","https://openalex.org/W2963989815","https://openalex.org/W2964231383","https://openalex.org/W2998216295","https://openalex.org/W3005000804","https://openalex.org/W3102449990","https://openalex.org/W3118608800","https://openalex.org/W4255949318","https://openalex.org/W4285719527","https://openalex.org/W6609194641","https://openalex.org/W6629461549","https://openalex.org/W6639204139","https://openalex.org/W6679792166","https://openalex.org/W6681608319","https://openalex.org/W6683645957","https://openalex.org/W6683738474","https://openalex.org/W6684191040","https://openalex.org/W6684372118","https://openalex.org/W6685133223","https://openalex.org/W6691459498","https://openalex.org/W6727323963","https://openalex.org/W6787972765","https://openalex.org/W7066667914"],"related_works":["https://openalex.org/W2163296013","https://openalex.org/W2743859443","https://openalex.org/W2326995835","https://openalex.org/W165915117","https://openalex.org/W2059402478","https://openalex.org/W2123347777","https://openalex.org/W4387804363","https://openalex.org/W2019547100","https://openalex.org/W2477150073","https://openalex.org/W2515493494"],"abstract_inverted_index":{"Deep":[0],"neural":[1,222],"networks":[2],"(DNNs)":[3],"have":[4,57],"demonstrated":[5],"impressive":[6],"performance":[7],"in":[8,69,95,100],"complex":[9],"machine":[10],"learning":[11],"tasks":[12],"such":[13,139],"as":[14,140],"image":[15],"classification":[16,44,89,187],"or":[17,45,193],"speech":[18],"recognition.":[19],"However,":[20],"due":[21],"to":[22,35,62,87,201],"their":[23],"multilayer":[24],"nonlinear":[25],"structure,":[26],"they":[27],"are":[28],"not":[29],"transparent,":[30],"i.e.,":[31],"it":[32],"is":[33,119,162],"hard":[34],"grasp":[36],"what":[37,179],"makes":[38],"them":[39],"arrive":[40,183],"at":[41,184],"a":[42,49,70,73,93,98,113,126,175,181,185],"particular":[43,186],"recognition":[46],"decision,":[47],"given":[48],"new":[50],"unseen":[51],"data":[52,157],"sample.":[53],"Recently,":[54],"several":[55],"approaches":[56],"been":[58],"proposed":[59,166],"enabling":[60],"one":[61],"understand":[63],"and":[64,91,154,172,205],"interpret":[65],"the":[66,80,88,104,151,164,190,194,213,221],"reasoning":[67],"embodied":[68],"DNN":[71,182],"for":[72,133,217],"single":[74],"test":[75],"image.":[76],"These":[77],"methods":[78,149],"quantify":[79],"\u201cimportance\u201d":[81],"of":[82,97,106,137,178,215,220],"individual":[83],"pixels":[84,138],"with":[85],"respect":[86],"decision":[90,188],"allow":[92],"visualization":[94],"terms":[96],"heatmap":[99],"pixel/input":[101],"space.":[102],"While":[103],"usefulness":[105],"heatmaps":[107,144,216],"can":[108],"be":[109],"judged":[110],"subjectively":[111],"by":[112,146],"human,":[114],"an":[115],"objective":[116],"quality":[117],"measure":[118],"missing.":[120],"In":[121],"this":[122,203],"paper,":[123],"we":[124,211],"present":[125],"general":[127],"methodology":[128],"based":[129],"on":[130,150],"region":[131],"perturbation":[132],"evaluating":[134],"ordered":[135],"collections":[136],"heatmaps.":[141],"We":[142,197],"compare":[143],"computed":[145],"three":[147],"different":[148],"SUN397,":[152],"ILSVRC2012,":[153],"MIT":[155],"Places":[156],"sets.":[158],"Our":[159],"main":[160],"result":[161,204],"that":[163],"recently":[165],"layer-wise":[167],"relevance":[168],"propagation":[169],"algorithm":[170],"qualitatively":[171],"quantitatively":[173],"provides":[174],"better":[176],"explanation":[177],"made":[180],"than":[189],"sensitivity-based":[191],"approach":[192],"deconvolution":[195],"method.":[196],"provide":[198],"theoretical":[199],"arguments":[200],"explain":[202],"discuss":[206],"its":[207],"practical":[208],"implications.":[209],"Finally,":[210],"investigate":[212],"use":[214],"unsupervised":[218],"assessment":[219],"network":[223],"performance.":[224]},"counts_by_year":[{"year":2026,"cited_by_count":59},{"year":2025,"cited_by_count":104},{"year":2024,"cited_by_count":141},{"year":2023,"cited_by_count":178},{"year":2022,"cited_by_count":152},{"year":2021,"cited_by_count":190},{"year":2020,"cited_by_count":141},{"year":2019,"cited_by_count":99},{"year":2018,"cited_by_count":69},{"year":2017,"cited_by_count":28},{"year":2016,"cited_by_count":10},{"year":2015,"cited_by_count":1}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}