{"id":"https://openalex.org/W3213356154","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","title":"Can NLI Models Verify QA Systems\u2019 Predictions?","display_name":"Can NLI Models Verify QA Systems\u2019 Predictions?","alternate_titles":[],"publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3213356154","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","mag":"3213356154"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2021.findings-emnlp.324","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","pdf_url":"https://aclanthology.org/2021.findings-emnlp.324.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W3213356154"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2021.findings-emnlp.324.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103130791","display_name":"Jifan Chen","orcid":"https://orcid.org/0000-0001-6814-0999","observed_orcids":["https://orcid.org/0000-0001-6814-0999"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Jifan Chen","raw_affiliation_strings":["Department of Computer Science The University of Texas at Austin","The University of Texas at Austin, Austin, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"The University of Texas at Austin, Austin, United States","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035142405","display_name":"Eunsol Choi","orcid":"https://orcid.org/0000-0003-3607-9104","observed_orcids":["https://orcid.org/0000-0003-3607-9104"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Eunsol Choi","raw_affiliation_strings":["Department of Computer Science The University of Texas at Austin","The University of Texas at Austin, Austin, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"The University of Texas at Austin, Austin, United States","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015133105","display_name":"Greg Durrett","orcid":"https://orcid.org/0000-0002-7061-7298","observed_orcids":["https://orcid.org/0000-0002-7061-7298"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"contributor_roles":[],"raw_author_name":"Greg Durrett","raw_affiliation_strings":["Department of Computer Science The University of Texas at Austin","The University of Texas at Austin, Austin, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"The University of Texas at Austin, Austin, United States","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I86519309"],"apc_list":null,"apc_paid":null,"fwci":4.0427,"has_fulltext":true,"cited_by_count":84,"citation_normalized_percentile":{"value":0.95275351,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"3841","last_page":"3854"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9009000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9009000062942505,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.06639999896287918,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0071000000461936,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language-inference","display_name":"natural language inference","score":0.9359999895095825},{"id":"https://openalex.org/keywords/question-answering-systems","display_name":"question answering systems","score":0.7450000047683716},{"id":"https://openalex.org/keywords/answer-verification","display_name":"answer verification","score":0.828000009059906},{"id":"https://openalex.org/keywords/confidence-estimation","display_name":"confidence estimation","score":0.7630000114440918},{"id":"https://openalex.org/keywords/decontextualization","display_name":"decontextualization","score":0.9559999704360962},{"id":"https://openalex.org/keywords/answer-correctness","display_name":"answer correctness","score":0.32899999618530273}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8760915994644165},{"id":"https://openalex.org/C2778023277","wikidata":"https://www.wikidata.org/wiki/Q321703","display_name":"Premise","level":2,"score":0.7717710733413696},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.707091748714447},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5776821970939636},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5441537499427795},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5253906846046448},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.472347617149353},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4494372308254242},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.44591641426086426},{"id":"https://openalex.org/C2780310539","wikidata":"https://www.wikidata.org/wiki/Q12547192","display_name":"Imperfect","level":2,"score":0.4421504735946655},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.42189720273017883},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3952132761478424},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09196650981903076},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/2021.findings-emnlp.324","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","pdf_url":"https://aclanthology.org/2021.findings-emnlp.324.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2104.08731","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.08731","pdf_url":"https://arxiv.org/pdf/2104.08731","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2104.08731","doi":"https://doi.org/10.48550/arxiv.2104.08731","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2104.08731","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3213356154","doi":null,"is_oa":false,"landing_page_url":"https://aclanthology.org/2021.findings-emnlp.324/","pdf_url":null,"source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Empirical Methods in Natural Language Processing","raw_type":null}],"best_oa_location":{"id":"doi:10.18653/v1/2021.findings-emnlp.324","doi":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.findings-emnlp.324","pdf_url":"https://aclanthology.org/2021.findings-emnlp.324.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2021","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1454387752","display_name":"RI: Small: Applying discrete reasoning steps in solving natural language processing tasks","funder_award_id":"1814522","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4713059963","display_name":null,"funder_award_id":"FA8750","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320338294","display_name":"Air Force Research Laboratory","ror":"https://ror.org/02e2egq70"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3213356154.pdf","grobid_xml":"https://content.openalex.org/works/W3213356154.grobid-xml"},"referenced_works_count":66,"referenced_works":["https://openalex.org/W1497080502","https://openalex.org/W1522301498","https://openalex.org/W1544827683","https://openalex.org/W1840435438","https://openalex.org/W1981208470","https://openalex.org/W2112644606","https://openalex.org/W2125436846","https://openalex.org/W2130158090","https://openalex.org/W2155173817","https://openalex.org/W2525127255","https://openalex.org/W2551396370","https://openalex.org/W2607892599","https://openalex.org/W2612431505","https://openalex.org/W2757675222","https://openalex.org/W2788496822","https://openalex.org/W2789566302","https://openalex.org/W2793978524","https://openalex.org/W2805083708","https://openalex.org/W2858627132","https://openalex.org/W2891308403","https://openalex.org/W2896457183","https://openalex.org/W2911430044","https://openalex.org/W2912924812","https://openalex.org/W2913222130","https://openalex.org/W2923014074","https://openalex.org/W2939380783","https://openalex.org/W2942128719","https://openalex.org/W2951211142","https://openalex.org/W2952862139","https://openalex.org/W2962989446","https://openalex.org/W2963310665","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963547127","https://openalex.org/W2963748441","https://openalex.org/W2963783970","https://openalex.org/W2963969878","https://openalex.org/W2964068236","https://openalex.org/W2964121744","https://openalex.org/W2965373594","https://openalex.org/W2970019270","https://openalex.org/W2970290563","https://openalex.org/W2970354142","https://openalex.org/W2977235550","https://openalex.org/W2979826702","https://openalex.org/W2981456979","https://openalex.org/W2981852735","https://openalex.org/W2988421999","https://openalex.org/W2994934025","https://openalex.org/W2997645422","https://openalex.org/W3007672467","https://openalex.org/W3014564055","https://openalex.org/W3035441651","https://openalex.org/W3049481998","https://openalex.org/W3081168214","https://openalex.org/W3092399494","https://openalex.org/W3098095571","https://openalex.org/W3100124407","https://openalex.org/W3105928338","https://openalex.org/W3128754101","https://openalex.org/W3168251909","https://openalex.org/W3169223343","https://openalex.org/W3173591450","https://openalex.org/W3173699110","https://openalex.org/W3174114019","https://openalex.org/W6893675597"],"related_works":["https://openalex.org/W4385571970","https://openalex.org/W4389523713","https://openalex.org/W2963903928","https://openalex.org/W4416189369","https://openalex.org/W4385573045","https://openalex.org/W3168251909","https://openalex.org/W2891308403","https://openalex.org/W2890191658","https://openalex.org/W73633278","https://openalex.org/W3092461730"],"abstract_inverted_index":{"To":[0],"build":[1],"robust":[2],"question":[3,78],"answering":[4],"systems,":[5],"we":[6,109],"need":[7],"the":[8,24,31,49,60,65,125,138,152,156,160,165,174,180,209,228,232,256,274,282,300,306],"ability":[9],"to":[10,14,41,53,58,64,75,114,218,265,287],"verify":[11],"whether":[12],"answers":[13],"questions":[15],"are":[16],"truly":[17],"correct,":[18,294],"not":[19,169],"just":[20],"\"good":[21],"enough\"":[22],"in":[23,208,231,255,277,305],"context":[25],"of":[26,33,128,140,173,179,281],"imperfect":[27],"QA":[28,86,106,116,130,153],"datasets.We":[29],"explore":[30],"use":[32],"natural":[34],"language":[35],"inference":[36],"(NLI)":[37],"as":[38,45,88],"a":[39,129,203,219,250,266,288],"way":[40],"achieve":[42],"this":[43],"goal,":[44],"NLI":[46,98,101,112,142,291],"inherently":[47],"requires":[48],"premise":[50],"(document":[51],"context)":[52],"contain":[54],"all":[55,171],"necessary":[56],"information":[57,296],"support":[59],"hypothesis":[61],"(proposed":[62],"answer":[63,158,166,283,309],"question).We":[66],"leverage":[67],"large":[68],"pretrained":[69],"models":[70,113],"and":[71,80,211,258],"recent":[72],"prior":[73],"datasets":[74,99],"construct":[76],"powerful":[77],"conversion":[79,286],"decontextualization":[81],"modules,":[82],"which":[83],"can":[84,110,147],"reformulate":[85],"instances":[87],"premise-hypothesis":[89],"pairs":[90],"with":[91,100],"very":[92],"high":[93],"reliability.Then,":[94],"by":[95,189,214,261],"combining":[96],"standard":[97],"examples":[102],"automatically":[103],"derived":[104],"from":[105],"training":[107],"data,":[108],"train":[111],"evaluate":[115],"systems'":[117],"proposed":[118],"answers.We":[119],"show":[120],"that":[121,145],"our":[122,141],"approach":[123],"improves":[124],"confidence":[126],"estimation":[127],"model":[131,143,154],"across":[132],"different":[133],"domains.Careful":[134],"manual":[135],"analysis":[136],"over":[137],"predictions":[139],"shows":[144],"it":[146],"further":[148],"identify":[149],"cases":[150],"where":[151],"produces":[155],"right":[157],"for":[159],"wrong":[161],"reason,":[162],"i.e.,":[163],"when":[164],"sentence":[167,284],"does":[168],"address":[170],"aspects":[172],"question.Context:":[175],"The":[176,185,194,238,240,278],"first":[177],"season":[178],"fantasy":[181],"comedy":[182],"television":[183],"series":[184,195,239],"Good":[186,233,241,279],"Place,":[187],"created":[188],"Michael":[190,215,262,298],"Schur,":[191],"aired":[192],"\u2026":[193,224],"focuses":[196,243],"on":[197,244],"Eleanor":[198,245],"Shellstrop":[199,246],"(Kristen":[200,247],"Bell)":[201,248],",":[202,249],"woman":[204,251],"who":[205,252],"wakes":[206,253],"up":[207,254],"afterlife":[210,257],"is":[212,259,293,303],"introduced":[213,260],"(Ted":[216,263],"Danson)":[217,264],"Heaven-like":[220,267],"utopia":[221,268],"he":[222,269],"designed":[223],"Question:":[225],"Who":[226],"plays":[227,273],"bad":[229,275,301],"guy":[230,276,302],"Place?Answer:":[234],"Ted":[235,271],"Danson":[236,272],"Premise:":[237],"Place":[242],"designed.Hypothesis:":[270],"Place.Decontextualization":[280],"Question":[285],"declarative":[289],"statement":[290],"ModelAnswer":[292],"but":[295],"about":[297],"being":[299],"missing":[304],"premiseNot":[307],"entailed,":[308],"rejected":[310]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3213356154","counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":17},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":18},{"year":2021,"cited_by_count":6}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2021-11-22T00:00:00"}
