{"id":"https://openalex.org/W4402640336","doi":"https://doi.org/10.1162/tacl_a_00692","title":"How Often Are Errors in Natural Language Reasoning Due to Paraphrastic Variability?","display_name":"How Often Are Errors in Natural Language Reasoning Due to Paraphrastic Variability?","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402640336","doi":"https://doi.org/10.1162/tacl_a_00692"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00692","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00692","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"http://dx.doi.org/10.1162/tacl_a_00692","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049725398","display_name":"Neha Srikanth","orcid":"https://orcid.org/0000-0001-7456-7470"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Neha Srikanth","raw_affiliation_strings":["Computer Science, University of Maryland, USA. nehasrik@umd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science, University of Maryland, USA. nehasrik@umd.edu","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078390032","display_name":"Marine Carpuat","orcid":"https://orcid.org/0000-0003-1693-0782"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marine Carpuat","raw_affiliation_strings":["Computer Science, University of Maryland, USA. marine@cs.umd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science, University of Maryland, USA. marine@cs.umd.edu","institution_ids":["https://openalex.org/I66946132"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082447472","display_name":"Rachel Rudinger","orcid":"https://orcid.org/0000-0002-5506-4701"},"institutions":[{"id":"https://openalex.org/I66946132","display_name":"University of Maryland, College Park","ror":"https://ror.org/047s2c258","country_code":"US","type":"education","lineage":["https://openalex.org/I66946132"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rachel Rudinger","raw_affiliation_strings":["Computer Science, University of Maryland, USA. rudinger@umd.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer Science, University of Maryland, USA. rudinger@umd.edu","institution_ids":["https://openalex.org/I66946132"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5049725398"],"corresponding_institution_ids":["https://openalex.org/I66946132"],"apc_list":null,"apc_paid":null,"fwci":0.9441,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.79742856,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"12","issue":null,"first_page":"1143","last_page":"1162"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.993399977684021,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8186893463134766},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.5608486533164978},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5308576226234436},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4986715316772461},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.49274662137031555},{"id":"https://openalex.org/keywords/history","display_name":"History","score":0.07726284861564636}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8186893463134766},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.5608486533164978},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5308576226234436},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4986715316772461},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.49274662137031555},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.07726284861564636},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00692","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00692","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:4d0dad1545a84ddca5e69b329cf1b4af","is_oa":true,"landing_page_url":"https://doaj.org/article/4d0dad1545a84ddca5e69b329cf1b4af","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00692","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1162/tacl_a_00692","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W1840435438","https://openalex.org/W1975879668","https://openalex.org/W2016172157","https://openalex.org/W2155322595","https://openalex.org/W2250539671","https://openalex.org/W2531638282","https://openalex.org/W2623751800","https://openalex.org/W2753392522","https://openalex.org/W2798665661","https://openalex.org/W2945735543","https://openalex.org/W2950470622","https://openalex.org/W2951286828","https://openalex.org/W2962736243","https://openalex.org/W2962843521","https://openalex.org/W2963101081","https://openalex.org/W2963126845","https://openalex.org/W2963159690","https://openalex.org/W2963748441","https://openalex.org/W2963846996","https://openalex.org/W2963918774","https://openalex.org/W2963961878","https://openalex.org/W2977235550","https://openalex.org/W2990138404","https://openalex.org/W3034850762","https://openalex.org/W3035252911","https://openalex.org/W3098613713","https://openalex.org/W3102749280","https://openalex.org/W3104041537","https://openalex.org/W3105928338","https://openalex.org/W3176894732","https://openalex.org/W3194676777","https://openalex.org/W3199958362","https://openalex.org/W3202712981","https://openalex.org/W3214559542","https://openalex.org/W4205509257","https://openalex.org/W4206657241","https://openalex.org/W4213168938","https://openalex.org/W4253067820","https://openalex.org/W4285194330","https://openalex.org/W4287889150","https://openalex.org/W4288089799","https://openalex.org/W4288262459","https://openalex.org/W4298041064","https://openalex.org/W4307079201","https://openalex.org/W4385565015","https://openalex.org/W4385570312","https://openalex.org/W6719819555","https://openalex.org/W6732580787","https://openalex.org/W6751744669","https://openalex.org/W6766937060","https://openalex.org/W6768299147","https://openalex.org/W6769627184","https://openalex.org/W6778883912","https://openalex.org/W6803557570","https://openalex.org/W6804097997","https://openalex.org/W6847076894","https://openalex.org/W6861236507"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W3204019825","https://openalex.org/W4226226396","https://openalex.org/W3153750606","https://openalex.org/W4308854837"],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"language":[2,56,122],"models":[3,29,58,147],"have":[4],"been":[5],"shown":[6],"to":[7,12,82,92],"behave":[8],"inconsistently":[9],"in":[10,89,153],"response":[11],"meaning-preserving":[13],"paraphrastic":[14,40,52,129,154],"inputs.":[15],"At":[16],"the":[17,22,37,51,61,67,74,83,128],"same":[18,68,75],"time,":[19],"researchers":[20],"evaluate":[21],"knowledge":[23],"and":[24,105,119,135],"reasoning":[25,57,108],"abilities":[26],"of":[27,39,54,63,73,85,102,113,131],"these":[28],"with":[30,141],"test":[31],"evaluations":[32],"that":[33,137],"do":[34],"not":[35,144],"disaggregate":[36],"effect":[38],"variability":[41],"on":[42,60,70,111],"performance.":[43],"We":[44,77],"propose":[45],"a":[46,64,86,100],"metric,":[47],"PC,":[48,96],"for":[49,117,151],"evaluating":[50],"consistency":[53,130,138],"natural":[55,121],"based":[59],"probability":[62],"model":[65,133],"achieving":[66],"correctness":[69,90],"two":[71],"paraphrases":[72],"problem.":[76],"mathematically":[78],"connect":[79],"this":[80],"metric":[81],"proportion":[84],"model\u2019s":[87],"variance":[88],"attributable":[91],"paraphrasing.":[93],"To":[94],"estimate":[95],"we":[97,126],"collect":[98],"ParaNlu,":[99,125],"dataset":[101],"7,782":[103],"human-written":[104],"validated":[106],"paraphrased":[107],"problems":[109],"constructed":[110],"top":[112],"existing":[114],"benchmark":[115],"datasets":[116],"defeasible":[118],"abductive":[120],"inference.1":[123],"Using":[124],"measure":[127],"several":[132],"classes":[134],"show":[136],"dramatically":[139],"increases":[140],"pretraining":[142],"but":[143],"fine-tuning.":[145],"All":[146],"tested":[148],"exhibited":[149],"room":[150],"improvement":[152],"consistency.":[155]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}