{"id":"https://openalex.org/W2962816513","doi":"https://doi.org/10.18653/v1/d18-1407","title":"Pathologies of Neural Models Make Interpretations Difficult","display_name":"Pathologies of Neural Models Make Interpretations Difficult","publication_year":2018,"publication_date":"2018-01-01","ids":{"openalex":"https://openalex.org/W2962816513","doi":"https://doi.org/10.18653/v1/d18-1407","mag":"2962816513"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d18-1407","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d18-1407","pdf_url":"https://www.aclweb.org/anthology/D18-1407.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D18-1407.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057958949","display_name":"Shi Feng","orcid":"https://orcid.org/0000-0002-2846-7652"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shi Feng","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110564361","display_name":"Eric Wallace","orcid":null},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Eric Wallace","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088386242","display_name":"Alvin Grissom","orcid":"https://orcid.org/0000-0002-6503-2703"},"institutions":[{"id":"https://openalex.org/I177753169","display_name":"Ursinus College","ror":"https://ror.org/02jqn4a25","country_code":"US","type":"education","lineage":["https://openalex.org/I177753169"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alvin Grissom II","raw_affiliation_strings":["Ursinus College"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Ursinus College","institution_ids":["https://openalex.org/I177753169"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082767919","display_name":"Mohit Iyyer","orcid":"https://orcid.org/0000-0001-7340-0804"},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mohit Iyyer","raw_affiliation_strings":["Allen Institute for Artificial Intelligence","UMass Amherst"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"UMass Amherst","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100707159","display_name":"Pedro Rodr\u00edguez","orcid":"https://orcid.org/0000-0001-8572-0725"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pedro Rodriguez","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081307846","display_name":"Jordan Boyd\u2010Graber","orcid":"https://orcid.org/0000-0002-7770-4431"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jordan Boyd-Graber","raw_affiliation_strings":["University of Maryland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Maryland","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":239,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3719","last_page":"3728"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7603254914283752},{"id":"https://openalex.org/keywords/counterintuitive","display_name":"Counterintuitive","score":0.6741472482681274},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6314353942871094},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.5933915972709656},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5174223184585571},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.5017824172973633},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.43599745631217957},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.430317759513855},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.42629873752593994},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.420434832572937},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13936010003089905}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7603254914283752},{"id":"https://openalex.org/C101097943","wikidata":"https://www.wikidata.org/wiki/Q5176983","display_name":"Counterintuitive","level":2,"score":0.6741472482681274},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6314353942871094},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.5933915972709656},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5174223184585571},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.5017824172973633},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.43599745631217957},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.430317759513855},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.42629873752593994},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.420434832572937},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13936010003089905},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/d18-1407","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d18-1407","pdf_url":"https://www.aclweb.org/anthology/D18-1407.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1804.07781","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1804.07781","pdf_url":"https://arxiv.org/pdf/1804.07781","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.18653/v1/d18-1407","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d18-1407","pdf_url":"https://www.aclweb.org/anthology/D18-1407.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G211012689","display_name":null,"funder_award_id":"HR0011-15-C-0113","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320307943","display_name":"Raytheon Company","ror":"https://ror.org/0354t7b78"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2962816513.pdf","grobid_xml":"https://content.openalex.org/works/W2962816513.grobid-xml"},"referenced_works_count":64,"referenced_works":["https://openalex.org/W1601924930","https://openalex.org/W1673923490","https://openalex.org/W1840435438","https://openalex.org/W1932198206","https://openalex.org/W1933349210","https://openalex.org/W1934849960","https://openalex.org/W1945616565","https://openalex.org/W2113459411","https://openalex.org/W2150165932","https://openalex.org/W2183341477","https://openalex.org/W2250473257","https://openalex.org/W2250539671","https://openalex.org/W2282821441","https://openalex.org/W2411480514","https://openalex.org/W2413794162","https://openalex.org/W2427527485","https://openalex.org/W2440159969","https://openalex.org/W2460937040","https://openalex.org/W2556096924","https://openalex.org/W2562979205","https://openalex.org/W2563399268","https://openalex.org/W2581377246","https://openalex.org/W2593833795","https://openalex.org/W2594475271","https://openalex.org/W2608030593","https://openalex.org/W2626967530","https://openalex.org/W2738015883","https://openalex.org/W2766047647","https://openalex.org/W2766912318","https://openalex.org/W2767899794","https://openalex.org/W2768037715","https://openalex.org/W2782630856","https://openalex.org/W2788403449","https://openalex.org/W2790415926","https://openalex.org/W2798139452","https://openalex.org/W2798836595","https://openalex.org/W2798858969","https://openalex.org/W2799007037","https://openalex.org/W2799194071","https://openalex.org/W2949197630","https://openalex.org/W2950734153","https://openalex.org/W2950761309","https://openalex.org/W2950864148","https://openalex.org/W2952113915","https://openalex.org/W2953084091","https://openalex.org/W2962736243","https://openalex.org/W2962843949","https://openalex.org/W2962851944","https://openalex.org/W2962985038","https://openalex.org/W2963029978","https://openalex.org/W2963126845","https://openalex.org/W2963207607","https://openalex.org/W2963323070","https://openalex.org/W2963661177","https://openalex.org/W2963748441","https://openalex.org/W2963834268","https://openalex.org/W2963969878","https://openalex.org/W2964040467","https://openalex.org/W2964121744","https://openalex.org/W2964153729","https://openalex.org/W2964159778","https://openalex.org/W2964178496","https://openalex.org/W2964212410","https://openalex.org/W4297798436"],"related_works":["https://openalex.org/W2809491669","https://openalex.org/W3005839474","https://openalex.org/W4379988549","https://openalex.org/W2139510495","https://openalex.org/W2016376779","https://openalex.org/W1992379025","https://openalex.org/W2768389068","https://openalex.org/W2403993643","https://openalex.org/W2118007841","https://openalex.org/W2765705957"],"abstract_inverted_index":{"One":[0],"way":[1],"to":[2,8,56,92,116,141],"interpret":[3],"neural":[4,85,153],"model":[5,44],"predictions":[6,129],"is":[7,35,49],"highlight":[9],"the":[10,20,41,52,61,73,78,87,97,111,118,127,165],"most":[11],"important":[12,75,101],"input":[13,24,39,68,181],"features-for":[14],"example,":[15],"a":[16,32],"heatmap":[17],"visualization":[18],"over":[19],"words":[21,89],"in":[22,43,151],"an":[23],"sentence.":[25],"In":[26],"existing":[27],"interpretation":[28,103],"methods":[29],"for":[30],"NLP,":[31],"word's":[33],"importance":[34],"determined":[36,99],"by":[37,51,102,167],"either":[38],"perturbation-measuring":[40],"decrease":[42],"confidence":[45,145],"when":[46],"that":[47,57],"word":[48,76],"removed-or":[50],"gradient":[53],"with":[54,108,130,156],"respect":[55],"word.":[58],"To":[59,133,159],"understand":[60],"limitations":[62],"of":[63,84,120],"these":[64,135],"methods,":[65],"we":[66,106,138,163],"use":[67],"reduction,":[69],"which":[70],"iteratively":[71],"removes":[72],"least":[74],"from":[77],"input.":[79],"This":[80],"exposes":[81],"pathological":[82,147],"behaviors":[83,148],"models:":[86],"remaining":[88],"appear":[90],"nonsensical":[91],"humans":[93],"and":[94,144],"are":[95],"not":[96],"ones":[98],"as":[100],"methods.":[104],"As":[105],"confirm":[107],"human":[109],"experiments,":[110],"reduced":[112,173],"examples":[113,143],"lack":[114],"information":[115],"support":[117],"prediction":[119],"any":[121],"label,":[122],"but":[123],"models":[124,154,166,176],"still":[125],"make":[126],"same":[128],"high":[131,169],"confidence.":[132],"explain":[134],"counterintuitive":[136],"results,":[137],"draw":[139],"connections":[140],"adversarial":[142],"calibration:":[146],"reveal":[149],"difficulties":[150],"interpreting":[152],"trained":[155],"maximum":[157],"likelihood.":[158],"mitigate":[160],"their":[161],"deficiencies,":[162],"fine-tune":[164],"encouraging":[168],"entropy":[170],"outputs":[171],"on":[172,186],"examples.":[174,188],"Fine-tuned":[175],"become":[177],"more":[178],"interpretable":[179],"under":[180],"reduction":[182],"without":[183],"accuracy":[184],"loss":[185],"regular":[187]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":42},{"year":2022,"cited_by_count":42},{"year":2021,"cited_by_count":43},{"year":2020,"cited_by_count":48},{"year":2019,"cited_by_count":36},{"year":2018,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}