{"id":"https://openalex.org/W2951286828","doi":"https://doi.org/10.18653/v1/p19-1334","title":"Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference","display_name":"Right for the Wrong Reasons: Diagnosing Syntactic Heuristics in Natural Language Inference","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2951286828","doi":"https://doi.org/10.18653/v1/p19-1334","mag":"2951286828"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1334","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1334","pdf_url":"https://www.aclweb.org/anthology/P19-1334.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1334.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081894510","display_name":"Tom McCoy","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"R. Thomas McCoy","raw_affiliation_strings":["Department of Cognitive Science, Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Cognitive Science, Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420"},"institutions":[{"id":"https://openalex.org/I175594653","display_name":"John Brown University","ror":"https://ror.org/02ct41q97","country_code":"US","type":"education","lineage":["https://openalex.org/I175594653"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Department of Computer Science, Brown University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Brown University","institution_ids":["https://openalex.org/I175594653"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5081824828","display_name":"Tal Linzen","orcid":"https://orcid.org/0000-0003-0435-6912"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tal Linzen","raw_affiliation_strings":["Department of Cognitive Science, Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Cognitive Science, Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":929,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3428","last_page":"3448"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.9253329634666443},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7519046068191528},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.7020182013511658},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6689738035202026},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6534625887870789},{"id":"https://openalex.org/keywords/subsequence","display_name":"Subsequence","score":0.6122123003005981},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5742089152336121},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5168721675872803},{"id":"https://openalex.org/keywords/test-set","display_name":"Test set","score":0.503337562084198},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5013446807861328},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.49715760350227356},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.48510730266571045},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.4843124449253082},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14072221517562866},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.13228362798690796}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.9253329634666443},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7519046068191528},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.7020182013511658},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6689738035202026},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6534625887870789},{"id":"https://openalex.org/C137877099","wikidata":"https://www.wikidata.org/wiki/Q1332977","display_name":"Subsequence","level":3,"score":0.6122123003005981},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5742089152336121},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5168721675872803},{"id":"https://openalex.org/C169903167","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Test set","level":2,"score":0.503337562084198},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5013446807861328},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.49715760350227356},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.48510730266571045},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.4843124449253082},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14072221517562866},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.13228362798690796},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C34388435","wikidata":"https://www.wikidata.org/wiki/Q2267362","display_name":"Bounded function","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1334","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1334","pdf_url":"https://www.aclweb.org/anthology/P19-1334.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1334","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1334","pdf_url":"https://www.aclweb.org/anthology/P19-1334.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7699999809265137,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G5280230632","display_name":"Graduate Research Fellowship Program (GRFP)","funder_award_id":"1746891","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320310145","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2951286828.pdf","grobid_xml":"https://content.openalex.org/works/W2951286828.grobid-xml"},"referenced_works_count":55,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W1894767800","https://openalex.org/W1897507002","https://openalex.org/W2002654918","https://openalex.org/W2097606805","https://openalex.org/W2101871539","https://openalex.org/W2117994434","https://openalex.org/W2130158090","https://openalex.org/W2143005196","https://openalex.org/W2228826686","https://openalex.org/W2308720496","https://openalex.org/W2413794162","https://openalex.org/W2506931122","https://openalex.org/W2515741950","https://openalex.org/W2549835527","https://openalex.org/W2561104024","https://openalex.org/W2608787653","https://openalex.org/W2770877945","https://openalex.org/W2771275742","https://openalex.org/W2788924045","https://openalex.org/W2791751435","https://openalex.org/W2798665661","https://openalex.org/W2799124508","https://openalex.org/W2887680905","https://openalex.org/W2888922637","https://openalex.org/W2889468083","https://openalex.org/W2896457183","https://openalex.org/W2899361264","https://openalex.org/W2910243263","https://openalex.org/W2950819771","https://openalex.org/W2962736243","https://openalex.org/W2962776659","https://openalex.org/W2962843521","https://openalex.org/W2963013168","https://openalex.org/W2963025830","https://openalex.org/W2963154734","https://openalex.org/W2963190614","https://openalex.org/W2963260436","https://openalex.org/W2963341956","https://openalex.org/W2963372062","https://openalex.org/W2963394326","https://openalex.org/W2963542100","https://openalex.org/W2963691697","https://openalex.org/W2963751529","https://openalex.org/W2963834860","https://openalex.org/W2963846996","https://openalex.org/W2963957386","https://openalex.org/W2963998676","https://openalex.org/W2964128011","https://openalex.org/W2964243640","https://openalex.org/W3099023595","https://openalex.org/W4288348198","https://openalex.org/W4288631803","https://openalex.org/W4298392964","https://openalex.org/W4300737232"],"related_works":["https://openalex.org/W4230102134","https://openalex.org/W2996768723","https://openalex.org/W4212895949","https://openalex.org/W4247866870","https://openalex.org/W2112772040","https://openalex.org/W3216774880","https://openalex.org/W2280422768","https://openalex.org/W3143197806","https://openalex.org/W4252555497","https://openalex.org/W2078387789"],"abstract_inverted_index":{"A":[0],"machine":[1],"learning":[2],"system":[3],"can":[4,142],"score":[5],"well":[6],"on":[7,14,105,115],"a":[8,81,109],"given":[9],"test":[10],"set":[11,84],"by":[12],"relying":[13],"heuristics":[15,98],"that":[16,50,102,118,127,138],"are":[17],"effective":[18],"for":[19,89,132],"frequent":[20],"example":[21],"types":[22],"but":[23],"break":[24],"down":[25],"in":[26,134,147],"more":[27],"challenging":[28],"cases.":[29],"We":[30,48,100,125],"study":[31],"this":[32,148],"issue":[33],"within":[34],"natural":[35],"language":[36],"inference":[37],"(NLI),":[38],"the":[39,60,64,68,97,139],"task":[40],"of":[41],"determining":[42],"whether":[43,73],"one":[44],"sentence":[45],"entails":[46],"another.":[47],"hypothesize":[49],"statistical":[51],"NLI":[52,90,135],"models":[53,74,103],"may":[54],"adopt":[55],"three":[56],"fallible":[57],"syntactic":[58],"heuristics:":[59],"lexical":[61],"overlap":[62],"heuristic,":[63,66],"subsequence":[65],"and":[67,137,144],"constituent":[69],"heuristic.":[70],"To":[71],"determine":[72],"have":[75,120],"adopted":[76,122],"these":[77,123],"heuristics,":[78],"we":[79],"introduce":[80],"controlled":[82],"evaluation":[83],"called":[85],"HANS":[86,140],"(Heuristic":[87],"Analysis":[88],"Systems),":[91],"which":[92],"contains":[93],"many":[94],"examples":[95],"where":[96],"fail.":[99],"find":[101],"trained":[104],"MNLI,":[106],"including":[107],"BERT,":[108],"state-of-the-art":[110],"model,":[111],"perform":[112],"very":[113],"poorly":[114],"HANS,":[116],"suggesting":[117],"they":[119],"indeed":[121],"heuristics.":[124],"conclude":[126],"there":[128],"is":[129],"substantial":[130],"room":[131],"improvement":[133],"systems,":[136],"dataset":[141],"motivate":[143],"measure":[145],"progress":[146],"area.":[149]},"counts_by_year":[{"year":2026,"cited_by_count":26},{"year":2025,"cited_by_count":39},{"year":2024,"cited_by_count":64},{"year":2023,"cited_by_count":198},{"year":2022,"cited_by_count":181},{"year":2021,"cited_by_count":226},{"year":2020,"cited_by_count":165},{"year":2019,"cited_by_count":30}],"updated_date":"2026-07-18T07:39:51.176621","created_date":"2025-10-10T00:00:00"}