{"id":"https://openalex.org/W2985347336","doi":"https://doi.org/10.1162/tacl_a_00293","title":"Inherent Disagreements in Human Textual Inferences","display_name":"Inherent Disagreements in Human Textual Inferences","publication_year":2019,"publication_date":"2019-11-01","ids":{"openalex":"https://openalex.org/W2985347336","doi":"https://doi.org/10.1162/tacl_a_00293","mag":"2985347336"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00293","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00293","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00293/1923015/tacl_a_00293.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00293/1923015/tacl_a_00293.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420"},"institutions":[{"id":"https://openalex.org/I175594653","display_name":"John Brown University","ror":"https://ror.org/02ct41q97","country_code":"US","type":"education","lineage":["https://openalex.org/I175594653"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Brown University","Brown University, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brown University","institution_ids":["https://openalex.org/I175594653"]},{"raw_affiliation_string":"Brown University, United States","institution_ids":["https://openalex.org/I175594653"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5011150187","display_name":"Tom Kwiatkowski","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tom Kwiatkowski","raw_affiliation_strings":["Google Research","Google Research, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Google Research, United States","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5011150187","https://openalex.org/A5053850863"],"corresponding_institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I175594653"],"apc_list":null,"apc_paid":null,"fwci":15.5308,"has_fulltext":false,"cited_by_count":197,"citation_normalized_percentile":{"value":0.99193516,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"7","issue":null,"first_page":"677","last_page":"694"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8442869186401367},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7422569990158081},{"id":"https://openalex.org/keywords/textual-entailment","display_name":"Textual entailment","score":0.7249754667282104},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.675263524055481},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6010491847991943},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5888462066650391},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5879989862442017},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.5526696443557739},{"id":"https://openalex.org/keywords/relation","display_name":"Relation (database)","score":0.5279088616371155},{"id":"https://openalex.org/keywords/language-understanding","display_name":"Language understanding","score":0.4818057417869568},{"id":"https://openalex.org/keywords/logical-consequence","display_name":"Logical consequence","score":0.4192522466182709},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4167424142360687},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.10935220122337341}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8442869186401367},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7422569990158081},{"id":"https://openalex.org/C95318506","wikidata":"https://www.wikidata.org/wiki/Q6588467","display_name":"Textual entailment","level":3,"score":0.7249754667282104},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.675263524055481},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6010491847991943},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5888462066650391},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5879989862442017},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.5526696443557739},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5279088616371155},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.4818057417869568},{"id":"https://openalex.org/C134752490","wikidata":"https://www.wikidata.org/wiki/Q374182","display_name":"Logical consequence","level":2,"score":0.4192522466182709},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4167424142360687},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.10935220122337341},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00293","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00293","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00293/1923015/tacl_a_00293.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:50d62531cffd425f9955eb79784aca64","is_oa":true,"landing_page_url":"https://doaj.org/article/50d62531cffd425f9955eb79784aca64","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 7, Pp 677-694 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00293","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00293","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00293/1923015/tacl_a_00293.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.6399999856948853,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":72,"referenced_works":["https://openalex.org/W77146693","https://openalex.org/W92973652","https://openalex.org/W1546109633","https://openalex.org/W1603598191","https://openalex.org/W1823558885","https://openalex.org/W1840435438","https://openalex.org/W1937062517","https://openalex.org/W1970381522","https://openalex.org/W1977970897","https://openalex.org/W2012942264","https://openalex.org/W2016415759","https://openalex.org/W2026261675","https://openalex.org/W2036285688","https://openalex.org/W2042333061","https://openalex.org/W2052506900","https://openalex.org/W2053670600","https://openalex.org/W2058959415","https://openalex.org/W2068737686","https://openalex.org/W2082802837","https://openalex.org/W2083195487","https://openalex.org/W2118508845","https://openalex.org/W2126185296","https://openalex.org/W2127849236","https://openalex.org/W2130158090","https://openalex.org/W2146349902","https://openalex.org/W2147638277","https://openalex.org/W2154524383","https://openalex.org/W2165086233","https://openalex.org/W2165094119","https://openalex.org/W2188631158","https://openalex.org/W2219598741","https://openalex.org/W2250493512","https://openalex.org/W2250638193","https://openalex.org/W2259472270","https://openalex.org/W2296506885","https://openalex.org/W2336933743","https://openalex.org/W2509607406","https://openalex.org/W2515782034","https://openalex.org/W2515796212","https://openalex.org/W2548036585","https://openalex.org/W2559655401","https://openalex.org/W2604272474","https://openalex.org/W2766137737","https://openalex.org/W2771275742","https://openalex.org/W2781528640","https://openalex.org/W2786028390","https://openalex.org/W2792932412","https://openalex.org/W2799915114","https://openalex.org/W2810877026","https://openalex.org/W2889468083","https://openalex.org/W2899923850","https://openalex.org/W2900065283","https://openalex.org/W2912150279","https://openalex.org/W2912409473","https://openalex.org/W2924484028","https://openalex.org/W2936763600","https://openalex.org/W2946809241","https://openalex.org/W2951286828","https://openalex.org/W2955797234","https://openalex.org/W2963025830","https://openalex.org/W2963341956","https://openalex.org/W2963706350","https://openalex.org/W2963846996","https://openalex.org/W3081916501","https://openalex.org/W3087981831","https://openalex.org/W3088788633","https://openalex.org/W3185337839","https://openalex.org/W4234216897","https://openalex.org/W4235505822","https://openalex.org/W4238634189","https://openalex.org/W4254506686","https://openalex.org/W4303105513"],"related_works":["https://openalex.org/W12963412","https://openalex.org/W2250460949","https://openalex.org/W3158371345","https://openalex.org/W3141423438","https://openalex.org/W2071098659","https://openalex.org/W2627035043","https://openalex.org/W4385571113","https://openalex.org/W1553238488","https://openalex.org/W3030695269","https://openalex.org/W2784994561"],"abstract_inverted_index":{"We":[0,11,42,71,89],"analyze":[1],"human\u2019s":[2],"disagreements":[3,16],"about":[4],"the":[5,35,46,63,80,102],"validity":[6],"of":[7,37,48,62,65,74,105],"natural":[8,56],"language":[9,57,85],"inferences.":[10],"show":[12,44],"that,":[13],"very":[14],"often,":[15],"are":[17],"not":[18,60],"dismissible":[19],"as":[20,26,32],"annotation":[21],"\u201cnoise\u201d,":[22],"but":[23],"rather":[24],"persist":[25],"we":[27,33],"collect":[28],"more":[29],"ratings":[30],"and":[31],"vary":[34],"amount":[36],"context":[38],"provided":[39],"to":[40,79,99],"raters.":[41],"further":[43],"that":[45,96],"type":[47,64],"uncertainty":[49,66],"captured":[50],"by":[51],"current":[52],"state-of-the-art":[53],"models":[54,98],"for":[55,91],"inference":[58,86],"is":[59],"reflective":[61],"present":[67],"in":[68,77],"human":[69,107],"disagreements.":[70],"discuss":[72],"implications":[73],"our":[75],"results":[76],"relation":[78],"recognizing":[81],"textual":[82],"entailment":[83],"(RTE)/natural":[84],"(NLI)":[87],"task.":[88],"argue":[90],"a":[92],"refined":[93],"evaluation":[94],"objective":[95],"requires":[97],"explicitly":[100],"capture":[101],"full":[103],"distribution":[104],"plausible":[106],"judgments.":[108]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":11},{"year":2024,"cited_by_count":13},{"year":2023,"cited_by_count":55},{"year":2022,"cited_by_count":35},{"year":2021,"cited_by_count":45},{"year":2020,"cited_by_count":29},{"year":2019,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}