{"id":"https://openalex.org/W4296711106","doi":"https://doi.org/10.1162/tacl_a_00506","title":"Evaluating Attribution in Dialogue Systems: The BEGIN Benchmark","display_name":"Evaluating Attribution in Dialogue Systems: The BEGIN Benchmark","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4296711106","doi":"https://doi.org/10.1162/tacl_a_00506"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00506","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00506","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00506/2043760/tacl_a_00506.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00506/2043760/tacl_a_00506.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049618494","display_name":"Nouha Dziri","orcid":null},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"Nouha Dziri","raw_affiliation_strings":["University of Alberta, Canada. dziri@cs.ualberta.ca"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Alberta, Canada. dziri@cs.ualberta.ca","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076412680","display_name":"Hannah Rashkin","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Hannah Rashkin","raw_affiliation_strings":["Google Research, USA. hrashkin@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research, USA. hrashkin@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081824828","display_name":"Tal Linzen","orcid":"https://orcid.org/0000-0003-0435-6912"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Tal Linzen","raw_affiliation_strings":["Google Research, USA. linzen@google.com","New York University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research, USA. linzen@google.com","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"New York University, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5020881468","display_name":"David Reitter","orcid":"https://orcid.org/0000-0002-7887-8257"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"David Reitter","raw_affiliation_strings":["Google Research, USA. reitter@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research, USA. reitter@google.com","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5020881468","https://openalex.org/A5049618494","https://openalex.org/A5076412680","https://openalex.org/A5081824828"],"corresponding_institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I154425047","https://openalex.org/I57206974"],"apc_list":null,"apc_paid":null,"fwci":4.4061,"has_fulltext":false,"cited_by_count":45,"citation_normalized_percentile":{"value":0.95104658,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"10","issue":null,"first_page":"1066","last_page":"1083"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8484740853309631},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.7239924073219299},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6834962368011475},{"id":"https://openalex.org/keywords/attribution","display_name":"Attribution","score":0.565257728099823},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5178587436676025},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.512939453125},{"id":"https://openalex.org/keywords/grounded-theory","display_name":"Grounded theory","score":0.5027449131011963},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4846561551094055},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4449566602706909},{"id":"https://openalex.org/keywords/qualitative-research","display_name":"Qualitative research","score":0.15792801976203918}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8484740853309631},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.7239924073219299},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6834962368011475},{"id":"https://openalex.org/C143299363","wikidata":"https://www.wikidata.org/wiki/Q900584","display_name":"Attribution","level":2,"score":0.565257728099823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5178587436676025},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.512939453125},{"id":"https://openalex.org/C156325361","wikidata":"https://www.wikidata.org/wiki/Q1152864","display_name":"Grounded theory","level":3,"score":0.5027449131011963},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4846561551094055},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4449566602706909},{"id":"https://openalex.org/C190248442","wikidata":"https://www.wikidata.org/wiki/Q839486","display_name":"Qualitative research","level":2,"score":0.15792801976203918},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00506","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00506","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00506/2043760/tacl_a_00506.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:f1dfe216285941edacd890f813e1ae44","is_oa":true,"landing_page_url":"https://doaj.org/article/f1dfe216285941edacd890f813e1ae44","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 10, Pp 1066-1083 (2022)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00506","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00506","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00506/2043760/tacl_a_00506.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.7599999904632568,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":87,"referenced_works":["https://openalex.org/W1517955056","https://openalex.org/W2095705004","https://openalex.org/W2101105183","https://openalex.org/W2492794003","https://openalex.org/W2612690371","https://openalex.org/W2768957049","https://openalex.org/W2781528640","https://openalex.org/W2843010082","https://openalex.org/W2888302696","https://openalex.org/W2891826200","https://openalex.org/W2951211142","https://openalex.org/W2951286828","https://openalex.org/W2952523122","https://openalex.org/W2962849707","https://openalex.org/W2962989446","https://openalex.org/W2963640662","https://openalex.org/W2963825865","https://openalex.org/W2963846996","https://openalex.org/W2964223283","https://openalex.org/W2965373594","https://openalex.org/W2972664115","https://openalex.org/W2973049837","https://openalex.org/W2979826702","https://openalex.org/W2988937804","https://openalex.org/W2999344102","https://openalex.org/W3022685184","https://openalex.org/W3034188538","https://openalex.org/W3034383590","https://openalex.org/W3034999214","https://openalex.org/W3035252911","https://openalex.org/W3035352537","https://openalex.org/W3035408261","https://openalex.org/W3035725000","https://openalex.org/W3099369213","https://openalex.org/W3099766584","https://openalex.org/W3101017384","https://openalex.org/W3106234277","https://openalex.org/W3153046263","https://openalex.org/W3153947101","https://openalex.org/W3155584966","https://openalex.org/W3155807546","https://openalex.org/W3159259047","https://openalex.org/W3159892921","https://openalex.org/W3170432046","https://openalex.org/W3171600947","https://openalex.org/W3174073137","https://openalex.org/W3176098057","https://openalex.org/W3177242492","https://openalex.org/W3186655327","https://openalex.org/W3186791274","https://openalex.org/W3207604732","https://openalex.org/W4205646617","https://openalex.org/W4205686084","https://openalex.org/W4212774754","https://openalex.org/W4224280227","https://openalex.org/W4224947967","https://openalex.org/W4226157795","https://openalex.org/W4237821014","https://openalex.org/W4287854593","https://openalex.org/W4287900772","https://openalex.org/W4288089799","https://openalex.org/W4288091035","https://openalex.org/W4288624561","https://openalex.org/W4378908626","https://openalex.org/W6631190155","https://openalex.org/W6640842352","https://openalex.org/W6674330103","https://openalex.org/W6680532216","https://openalex.org/W6682631176","https://openalex.org/W6739901393","https://openalex.org/W6755699846","https://openalex.org/W6758737392","https://openalex.org/W6759693536","https://openalex.org/W6761205521","https://openalex.org/W6766673545","https://openalex.org/W6768028577","https://openalex.org/W6769492440","https://openalex.org/W6769627184","https://openalex.org/W6772715161","https://openalex.org/W6779068807","https://openalex.org/W6779132830","https://openalex.org/W6797816506","https://openalex.org/W6810104136","https://openalex.org/W6810584998","https://openalex.org/W6810934296","https://openalex.org/W6811640613","https://openalex.org/W6949549905"],"related_works":["https://openalex.org/W3113091479","https://openalex.org/W2162899405","https://openalex.org/W941090075","https://openalex.org/W2044987316","https://openalex.org/W3134374554","https://openalex.org/W2237480245","https://openalex.org/W2075065631","https://openalex.org/W2519167559","https://openalex.org/W4311248832","https://openalex.org/W4386113923"],"abstract_inverted_index":{"Abstract":[0],"Knowledge-grounded":[1],"dialogue":[2,56,61,67],"systems":[3,62],"powered":[4],"by":[5,59],"large":[6],"language":[7],"models":[8,26],"often":[9],"generate":[10],"responses":[11,80,113],"that,":[12],"while":[13],"fluent,":[14],"are":[15],"not":[16,29,108],"attributable":[17,111],"to":[18,76,84,93],"a":[19],"relevant":[20],"source":[21,124],"of":[22,50],"information.":[23,88],"Progress":[24],"towards":[25],"that":[27,36,100],"do":[28,107],"exhibit":[30],"this":[31,42],"issue":[32],"requires":[33],"evaluation":[34,96,137],"metrics":[35,102,138],"can":[37,81],"quantify":[38],"its":[39],"prevalence.":[40],"To":[41],"end,":[43],"we":[44],"introduce":[45],"the":[46,74,78,85,122,130],"Benchmark":[47],"for":[48,132,139],"Evaluation":[49],"Grounded":[51],"INteraction":[52],"(Begin),":[53],"comprising":[54],"12k":[55],"turns":[57],"generated":[58],"neural":[60],"trained":[63],"on":[64,104],"three":[65],"knowledge-grounded":[66,140],"corpora.":[68],"We":[69,89,98,142],"collect":[70],"human":[71],"annotations":[72],"assessing":[73],"extent":[75],"which":[77],"models\u2019":[79],"be":[82],"attributed":[83],"given":[86],"background":[87],"then":[90],"use":[91],"Begin":[92,144],"analyze":[94],"eight":[95],"metrics.":[97],"find":[99],"these":[101],"rely":[103],"spurious":[105],"correlations,":[106],"reliably":[109],"distinguish":[110],"abstractive":[112],"from":[114],"unattributable":[115],"ones,":[116],"and":[117,135],"perform":[118],"substantially":[119],"worse":[120],"when":[121],"knowledge":[123],"is":[125],"longer.":[126],"Our":[127],"findings":[128],"underscore":[129],"need":[131],"more":[133],"sophisticated":[134],"robust":[136],"dialogue.":[141],"make":[143],"publicly":[145],"available":[146],"at":[147],"https://github.com/google/BEGIN-dataset.":[148]},"counts_by_year":[{"year":2026,"cited_by_count":7},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":1}],"updated_date":"2026-08-12T21:12:35.861297","created_date":"2025-10-10T00:00:00"}