{"id":"https://openalex.org/W4398757454","doi":"https://doi.org/10.1162/tacl_a_00667","title":"Evaluating Correctness and Faithfulness of Instruction-Following Models for Question Answering","display_name":"Evaluating Correctness and Faithfulness of Instruction-Following Models for Question Answering","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4398757454","doi":"https://doi.org/10.1162/tacl_a_00667"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00667","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00667","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00667/2374170/tacl_a_00667.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00667/2374170/tacl_a_00667.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049387196","display_name":"Vaibhav Adlakha","orcid":null},"institutions":[{"id":"https://openalex.org/I4210099137","display_name":"Research Canada","ror":"https://ror.org/0103eqz62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210099137"]},{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I4210167196","display_name":"Minnow Environmental (Canada)","ror":"https://ror.org/05petyd50","country_code":"CA","type":"company","lineage":["https://openalex.org/I4210167196"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Vaibhav Adlakha","raw_affiliation_strings":["Mila, McGill University, Canada. vaibhav.adlakha@mila.quebec","ServiceNow Research, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila, McGill University, Canada. vaibhav.adlakha@mila.quebec","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]},{"raw_affiliation_string":"ServiceNow Research, Canada","institution_ids":["https://openalex.org/I4210099137","https://openalex.org/I4210167196"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063732194","display_name":"Parishad BehnamGhader","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Parishad BehnamGhader","raw_affiliation_strings":["Mila, McGill University, Canada. parishad.behnamghader@mila.quebec"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila, McGill University, Canada. parishad.behnamghader@mila.quebec","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101687671","display_name":"Xing Han L\u00f9","orcid":"https://orcid.org/0000-0001-9027-8425"},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Xing Han Lu","raw_affiliation_strings":["Mila, McGill University, Canada. xing-han.lu@mila.quebec"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila, McGill University, Canada. xing-han.lu@mila.quebec","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058072829","display_name":"Nicholas Meade","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Nicholas Meade","raw_affiliation_strings":["Mila, McGill University, Canada. nicholas.meade@mila.quebec"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila, McGill University, Canada. nicholas.meade@mila.quebec","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102886212","display_name":"Siva Reddy","orcid":null},"institutions":[{"id":"https://openalex.org/I109736498","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95","country_code":"CA","type":"facility","lineage":["https://openalex.org/I109736498"]},{"id":"https://openalex.org/I4210099137","display_name":"Research Canada","ror":"https://ror.org/0103eqz62","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210099137"]},{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I4210167196","display_name":"Minnow Environmental (Canada)","ror":"https://ror.org/05petyd50","country_code":"CA","type":"company","lineage":["https://openalex.org/I4210167196"]},{"id":"https://openalex.org/I5023651","display_name":"McGill University","ror":"https://ror.org/01pxwe438","country_code":"CA","type":"education","lineage":["https://openalex.org/I5023651"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Siva Reddy","raw_affiliation_strings":["Facebook CIFAR AI Chair, Canada","Mila, McGill University, Canada. siva.reddy@mila.quebec","ServiceNow Research, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Facebook CIFAR AI Chair, Canada","institution_ids":["https://openalex.org/I109736498"]},{"raw_affiliation_string":"Mila, McGill University, Canada. siva.reddy@mila.quebec","institution_ids":["https://openalex.org/I4210164802","https://openalex.org/I5023651"]},{"raw_affiliation_string":"ServiceNow Research, Canada","institution_ids":["https://openalex.org/I4210099137","https://openalex.org/I4210167196"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5049387196","https://openalex.org/A5058072829","https://openalex.org/A5063732194","https://openalex.org/A5101687671","https://openalex.org/A5102886212"],"corresponding_institution_ids":["https://openalex.org/I109736498","https://openalex.org/I4210099137","https://openalex.org/I4210164802","https://openalex.org/I4210167196","https://openalex.org/I5023651"],"apc_list":null,"apc_paid":null,"fwci":16.5535,"has_fulltext":false,"cited_by_count":70,"citation_normalized_percentile":{"value":0.99347189,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"681","last_page":"699"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9714000225067139,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11715","display_name":"Education and Critical Thinking Development","score":0.9659000039100647,"subfield":{"id":"https://openalex.org/subfields/3304","display_name":"Education"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.9042854309082031},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.874668300151825},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.561216413974762},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.416491836309433},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37205660343170166},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.30445289611816406}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.9042854309082031},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.874668300151825},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.561216413974762},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.416491836309433},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37205660343170166},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30445289611816406}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00667","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00667","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00667/2374170/tacl_a_00667.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:d507fd82b9fb490b9ad7c0c05cb5f0cd","is_oa":true,"landing_page_url":"https://doaj.org/article/d507fd82b9fb490b9ad7c0c05cb5f0cd","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00667","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00667","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00667/2374170/tacl_a_00667.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8399999737739563,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4398757454.pdf"},"referenced_works_count":69,"referenced_works":["https://openalex.org/W1979354511","https://openalex.org/W2600463316","https://openalex.org/W2612690371","https://openalex.org/W2889787757","https://openalex.org/W2912924812","https://openalex.org/W2951434086","https://openalex.org/W2963748441","https://openalex.org/W2964223283","https://openalex.org/W3099700870","https://openalex.org/W3153947101","https://openalex.org/W3155807546","https://openalex.org/W3156789018","https://openalex.org/W3176098057","https://openalex.org/W3196731672","https://openalex.org/W3201145629","https://openalex.org/W4212774754","https://openalex.org/W4221152111","https://openalex.org/W4226059645","https://openalex.org/W4226157795","https://openalex.org/W4229005866","https://openalex.org/W4287854593","https://openalex.org/W4288089799","https://openalex.org/W4296711106","https://openalex.org/W4297435087","https://openalex.org/W4307079201","https://openalex.org/W4312205996","https://openalex.org/W4313459318","https://openalex.org/W4322718191","https://openalex.org/W4362707064","https://openalex.org/W4378942694","https://openalex.org/W4384918448","https://openalex.org/W4385569780","https://openalex.org/W4385570384","https://openalex.org/W4385570777","https://openalex.org/W4385571232","https://openalex.org/W4385571412","https://openalex.org/W4385572634","https://openalex.org/W4385572714","https://openalex.org/W4385572845","https://openalex.org/W4389519254","https://openalex.org/W4391876619","https://openalex.org/W4401042773","https://openalex.org/W4402671800","https://openalex.org/W6628905179","https://openalex.org/W6678262379","https://openalex.org/W6682631176","https://openalex.org/W6739585900","https://openalex.org/W6761205521","https://openalex.org/W6769627184","https://openalex.org/W6777615688","https://openalex.org/W6778883912","https://openalex.org/W6784180246","https://openalex.org/W6787719664","https://openalex.org/W6800875267","https://openalex.org/W6802669662","https://openalex.org/W6810152653","https://openalex.org/W6810738896","https://openalex.org/W6810934296","https://openalex.org/W6811340617","https://openalex.org/W6843135089","https://openalex.org/W6847076894","https://openalex.org/W6847306236","https://openalex.org/W6849504698","https://openalex.org/W6850625674","https://openalex.org/W6851960618","https://openalex.org/W6852754783","https://openalex.org/W6852947401","https://openalex.org/W6853859572","https://openalex.org/W6854866820"],"related_works":["https://openalex.org/W1667647204","https://openalex.org/W2404647514","https://openalex.org/W4247536566","https://openalex.org/W2018477250","https://openalex.org/W3119814709","https://openalex.org/W4241418540","https://openalex.org/W1508895727","https://openalex.org/W2725786787","https://openalex.org/W4283160672","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Abstract":[0],"Instruction-following":[1],"models":[2,25,40,72,133,137,171],"are":[3],"attractive":[4],"alternatives":[5],"to":[6,21,29,42,136,146],"fine-tuned":[7,139],"approaches":[8],"for":[9,61,110,130,140,172],"question":[10],"answering":[11],"(QA).":[12],"By":[13],"simply":[14],"prepending":[15],"relevant":[16],"documents":[17],"and":[18,33,58,85,101,113,115,155,176],"an":[19],"instruction":[20],"their":[22,159],"input,":[23],"these":[24,39],"can":[26],"be":[27],"adapted":[28],"various":[30],"information":[31,82,90],"domains":[32],"tasks":[34],"without":[35],"additional":[36],"training.":[37],"However,":[38,143],"tend":[41],"produce":[43],"verbose":[44],"responses":[45],"with":[46,123],"supplementary":[47],"information,":[48],"which":[49],"makes":[50],"traditional":[51,108],"QA":[52],"metrics":[53,109,119],"like":[54],"exact":[55],"match":[56],"(EM)":[57],"F1":[59],"unreliable":[60],"accurately":[62,147],"quantifying":[63],"model":[64],"performance.":[65],"In":[66],"this":[67],"work,":[68],"we":[69,103],"evaluate":[70],"instruction-following":[71,132,170],"along":[73],"two":[74],"fronts:":[75],"1)":[76],"how":[77],"well":[78],"they":[79,88,144],"satisfy":[80],"user\u2019s":[81],"need":[83],"(correctness),":[84],"2)":[86],"whether":[87],"disseminate":[89],"supported":[91],"by":[92,98],"the":[93,105,149,152],"provided":[94,153],"knowledge":[95,154],"(faithfulness).":[96],"Guided":[97],"human":[99,124,177],"evaluation":[100,168],"analysis,":[102],"highlight":[104],"shortcomings":[106],"of":[107,151,169],"both":[111],"correctness":[112],"faithfulness":[114],"propose":[116],"simple":[117],"token-overlap":[118],"that":[120,129,141],"correlate":[121],"highly":[122],"judgments.":[125],"Our":[126,174],"analysis":[127],"reveals":[128],"correctness,":[131],"perform":[134],"comparably":[135],"specifically":[138],"task.":[142],"struggle":[145],"judge":[148],"relevance":[150],"often":[156],"hallucinate":[157],"in":[158],"responses.":[160],"We":[161],"hope":[162],"our":[163],"work":[164],"encourages":[165],"more":[166],"holistic":[167],"QA.":[173],"code":[175],"annotation":[178],"data":[179],"is":[180],"available":[181],"at":[182],"https://github.com/McGill-NLP/instruct-qa.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":36},{"year":2024,"cited_by_count":18},{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}