{"id":"https://openalex.org/W4296631020","doi":"https://doi.org/10.3233/faia220190","title":"A Song of (Dis)agreement: Evaluating the Evaluation of Explainable Artificial Intelligence in Natural Language Processing","display_name":"A Song of (Dis)agreement: Evaluating the Evaluation of Explainable Artificial Intelligence in Natural Language Processing","publication_year":2022,"publication_date":"2022-09-19","ids":{"openalex":"https://openalex.org/W4296631020","doi":"https://doi.org/10.3233/faia220190"},"language":"en","primary_location":{"id":"doi:10.3233/faia220190","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia220190","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA220190","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA220190","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029961049","display_name":"Michael J. Neely","orcid":"https://orcid.org/0000-0003-3524-1587"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Michael Neely","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066100494","display_name":"Stefan F. Schouten","orcid":null},"institutions":[{"id":"https://openalex.org/I865915315","display_name":"Vrije Universiteit Amsterdam","ror":"https://ror.org/008xxew50","country_code":"NL","type":"education","lineage":["https://openalex.org/I865915315"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Stefan F. Schouten","raw_affiliation_strings":["University of Amsterdam","Vrije Universiteit Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"Vrije Universiteit Amsterdam","institution_ids":["https://openalex.org/I865915315"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056994081","display_name":"Maurits Bleeker","orcid":null},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maurits Bleeker","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018957074","display_name":"Ana Lu\u010di\u0107","orcid":"https://orcid.org/0000-0002-5509-445X"},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Ana Lucic","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.3876,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.90902962,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9126999974250793,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6274401545524597},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6025579571723938},{"id":"https://openalex.org/keywords/attribution","display_name":"Attribution","score":0.6005223989486694},{"id":"https://openalex.org/keywords/intuition","display_name":"Intuition","score":0.5954739451408386},{"id":"https://openalex.org/keywords/authorship-attribution","display_name":"Authorship attribution","score":0.5430887937545776},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.5347468256950378},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.448910117149353},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.4381178915500641},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42093849182128906},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.41988301277160645},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.36317014694213867},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.3362835943698883},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.18615314364433289},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.16408967971801758},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12689587473869324}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6274401545524597},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6025579571723938},{"id":"https://openalex.org/C143299363","wikidata":"https://www.wikidata.org/wiki/Q900584","display_name":"Attribution","level":2,"score":0.6005223989486694},{"id":"https://openalex.org/C132010649","wikidata":"https://www.wikidata.org/wiki/Q189222","display_name":"Intuition","level":2,"score":0.5954739451408386},{"id":"https://openalex.org/C3020202489","wikidata":"https://www.wikidata.org/wiki/Q2032038","display_name":"Authorship attribution","level":2,"score":0.5430887937545776},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.5347468256950378},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.448910117149353},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4381178915500641},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42093849182128906},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.41988301277160645},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.36317014694213867},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3362835943698883},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.18615314364433289},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.16408967971801758},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12689587473869324},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3233/faia220190","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia220190","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA220190","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},{"id":"pmh:oai:dare.uva.nl:publications/a6bd6a19-44ff-4666-85ed-7088cfc7cda7","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/a-song-of-disagreement-evaluating-the-evaluation-of-explainable-artificial-intelligence-in-natural-language-processing(a6bd6a19-44ff-4666-85ed-7088cfc7cda7).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neely, M, Schouten, S F, Bleeker, M & Lucic, A 2022, A Song of (Dis)agreement: Evaluating the Evaluation of Explainable Artificial Intelligence in Natural Language Processing. in S Schlobach, M P\u00e9rez-Ortiz & M Tielman (eds), HHAI2022: Augmenting Human Intellect : Proceedings of the 1st International Conference on Hybrid Human-Artificial Intelligence. Frontiers in Artificial Intelligence and Applications, vol. 354, Amsterdam, pp. 60-78, 1st International Conference on Hybrid Human-Artificial Intelligence, HHAI 2022, Amsterdam, Netherlands, 13/06/22. https://doi.org/10.3233/FAIA220190","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:dare.uva.nl:openaire/a6bd6a19-44ff-4666-85ed-7088cfc7cda7","is_oa":false,"landing_page_url":"https://hdl.handle.net/11245.1/a6bd6a19-44ff-4666-85ed-7088cfc7cda7","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Neely, M, Schouten, S F, Bleeker, M & Lucic, A 2022, A Song of (Dis)agreement: Evaluating the Evaluation of Explainable Artificial Intelligence in Natural Language Processing. in S Schlobach, M P\u00e9rez-Ortiz & M Tielman (eds), HHAI2022: Augmenting Human Intellect : Proceedings of the 1st International Conference on Hybrid Human-Artificial Intelligence. Frontiers in Artificial Intelligence and Applications, vol. 354, Amsterdam, pp. 60-78, 1st International Conference on Hybrid Human-Artificial Intelligence, HHAI 2022, Amsterdam, Netherlands, 13/06/22. https://doi.org/10.3233/FAIA220190","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.3233/faia220190","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia220190","pdf_url":"https://ebooks.iospress.nl/pdf/doi/10.3233/FAIA220190","source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.7900000214576721}],"awards":[{"id":"https://openalex.org/G3032427513","display_name":null,"funder_award_id":"652.001.003","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4296631020.pdf","grobid_xml":"https://content.openalex.org/works/W4296631020.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W1514535095","https://openalex.org/W1840435438","https://openalex.org/W1985514943","https://openalex.org/W1996796871","https://openalex.org/W2064675550","https://openalex.org/W2113459411","https://openalex.org/W2133280805","https://openalex.org/W2133564696","https://openalex.org/W2282821441","https://openalex.org/W2552396116","https://openalex.org/W2562979205","https://openalex.org/W2594475271","https://openalex.org/W2594633041","https://openalex.org/W2605409611","https://openalex.org/W2618851150","https://openalex.org/W2748003190","https://openalex.org/W2807124908","https://openalex.org/W2891555348","https://openalex.org/W2901277930","https://openalex.org/W2908510526","https://openalex.org/W2934842096","https://openalex.org/W2950768109","https://openalex.org/W2952524552","https://openalex.org/W2953903036","https://openalex.org/W2963341956","https://openalex.org/W2963691697","https://openalex.org/W2964159778","https://openalex.org/W2970726176","https://openalex.org/W2972324944","https://openalex.org/W2977944219","https://openalex.org/W2978017171","https://openalex.org/W2980282514","https://openalex.org/W2986218154","https://openalex.org/W3030406438","https://openalex.org/W3034491534","https://openalex.org/W3034834827","https://openalex.org/W3034917890","https://openalex.org/W3035281110","https://openalex.org/W3035422918","https://openalex.org/W3035503910","https://openalex.org/W3037212200","https://openalex.org/W3043146651","https://openalex.org/W3092292656","https://openalex.org/W3101155149","https://openalex.org/W3104313653","https://openalex.org/W3116662254","https://openalex.org/W3138819813","https://openalex.org/W3165941178","https://openalex.org/W3166235221","https://openalex.org/W3173447414","https://openalex.org/W3177333623","https://openalex.org/W3186950920","https://openalex.org/W3197047205","https://openalex.org/W3207937903","https://openalex.org/W4206917460","https://openalex.org/W4226318446","https://openalex.org/W4249616221","https://openalex.org/W4288025980","https://openalex.org/W4377825911","https://openalex.org/W4385245566","https://openalex.org/W6631190155"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2296853219","https://openalex.org/W2354866896","https://openalex.org/W1970186044","https://openalex.org/W2083986287","https://openalex.org/W2393417858","https://openalex.org/W207304934","https://openalex.org/W4377703168","https://openalex.org/W1560657467"],"abstract_inverted_index":{"There":[0],"has":[1,42],"been":[2,45],"significant":[3],"debate":[4],"in":[5],"the":[6,49,69,116,126,135,142,149,179,213,220],"NLP":[7,87],"community":[8,180],"about":[9],"whether":[10],"or":[11,118],"not":[12,105],"attention":[13],"weights":[14],"can":[15],"be":[16],"used":[17],"as":[18,40,186],"an":[19,187],"explanation":[20,164,203],"\u2013":[21],"a":[22,33,207],"mechanism":[23],"for":[24,32,134,190,219],"interpreting":[25],"how":[26,156],"important":[27],"each":[28],"input":[29],"token":[30],"is":[31],"particular":[34,221],"prediction.":[35],"The":[36],"validity":[37,150],"of":[38,86,115,125,151],"\u201cattention":[39],"explanation\u201d":[41],"so":[43],"far":[44],"evaluated":[46],"by":[47],"computing":[48],"rank":[50,70,184],"correlation":[51,71,185],"between":[52,72],"attention-based":[53,81,102,152,191],"explanations":[54,59,103,128,153,214],"and":[55,79,89,197,205],"existing":[56,161],"feature":[57,76,111,162],"attribution":[58,77,112,163],"using":[60,172,183],"LSTM-based":[61],"models.":[62,98],"In":[63],"our":[64],"work,":[65],"we":[66,121,146,176],"(i)":[67],"compare":[68],"five":[73,170],"more":[74],"recent":[75,110],"methods":[78,204],"two":[80,84,173],"methods,":[82,113],"on":[83,155,169],"types":[85],"tasks,":[88],"(ii)":[90],"extend":[91],"this":[92],"analysis":[93],"to":[94,140,210],"also":[95],"include":[96],"transformer-based":[97,136],"We":[99,193],"find":[100,122],"that":[101,123,145,178,195],"do":[104],"correlate":[106,129,159],"strongly":[107,130],"with":[108,131,160,216],"any":[109],"regardless":[114],"model":[117],"task.":[119],"Furthermore,":[120],"none":[124],"tested":[127],"one":[132],"another":[133],"model,":[137],"leading":[138],"us":[139],"question":[141],"underlying":[143],"assumption":[144],"should":[147,181,199],"measure":[148],"based":[154],"well":[157],"they":[158],"methods.":[165],"After":[166],"conducting":[167],"experiments":[168],"datasets":[171],"different":[174],"models,":[175],"argue":[177],"stop":[182],"evaluation":[188],"metric":[189],"explanations.":[192],"suggest":[194],"researchers":[196],"practitioners":[198],"instead":[200],"test":[201],"various":[202],"employ":[206],"human-in-the-loop":[208],"process":[209],"determine":[211],"if":[212],"align":[215],"human":[217],"intuition":[218],"use":[222],"case":[223],"at":[224],"hand.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}