{"id":"https://openalex.org/W2948628052","doi":"https://doi.org/10.18653/v1/p19-1483","title":"Handling Divergent Reference Texts when Evaluating Table-to-Text Generation","display_name":"Handling Divergent Reference Texts when Evaluating Table-to-Text Generation","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2948628052","doi":"https://doi.org/10.18653/v1/p19-1483","mag":"2948628052"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1483","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1483","pdf_url":"https://www.aclweb.org/anthology/P19-1483.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1483.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055033421","display_name":"Bhuwan Dhingra","orcid":"https://orcid.org/0000-0002-6874-9515"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhuwan Dhingra","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Carnegie Mellon University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014855689","display_name":"Manaal Faruqui","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Manaal Faruqui","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109937799","display_name":"Ankur P. Parikh","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ankur Parikh","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076904467","display_name":"Ming\u2010Wei Chang","orcid":"https://orcid.org/0000-0002-0137-8895"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ming-Wei Chang","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021055773","display_name":"Dipanjan Das","orcid":"https://orcid.org/0000-0003-2325-0646"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dipanjan Das","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051617344","display_name":"William W. Cohen","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"William Cohen","raw_affiliation_strings":["Carnegie Mellon University  Google Research","Google,,,,,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University  Google Research","institution_ids":["https://openalex.org/I1291425158","https://openalex.org/I74973139"]},{"raw_affiliation_string":"Google,,,,,","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4884","last_page":"4895"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9973999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7575136423110962},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6700255274772644},{"id":"https://openalex.org/keywords/table","display_name":"Table (database)","score":0.644761323928833},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6394175291061401},{"id":"https://openalex.org/keywords/text-generation","display_name":"Text generation","score":0.6108261942863464},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5599300265312195},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.5325438380241394},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.5270241498947144},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5096333026885986},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.4137669801712036},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.2705453336238861},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.11298403143882751},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10743716359138489},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.08788380026817322}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7575136423110962},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6700255274772644},{"id":"https://openalex.org/C45235069","wikidata":"https://www.wikidata.org/wiki/Q278425","display_name":"Table (database)","level":2,"score":0.644761323928833},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6394175291061401},{"id":"https://openalex.org/C2985684807","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Text generation","level":2,"score":0.6108261942863464},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5599300265312195},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.5325438380241394},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.5270241498947144},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5096333026885986},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.4137669801712036},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2705453336238861},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.11298403143882751},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10743716359138489},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.08788380026817322},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/p19-1483","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1483","pdf_url":"https://www.aclweb.org/anthology/P19-1483.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1906.01081","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.01081","pdf_url":"https://arxiv.org/pdf/1906.01081","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2948628052","is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1906.01081","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1906.01081","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1906.01081","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1483","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1483","pdf_url":"https://www.aclweb.org/anthology/P19-1483.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"No poverty","id":"https://metadata.un.org/sdg/1","score":0.6299999952316284}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2948628052.pdf","grobid_xml":"https://content.openalex.org/works/W2948628052.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W1489525520","https://openalex.org/W1552182777","https://openalex.org/W1570031249","https://openalex.org/W1763968285","https://openalex.org/W1956340063","https://openalex.org/W1985610876","https://openalex.org/W2003170434","https://openalex.org/W2068390867","https://openalex.org/W2078861931","https://openalex.org/W2101105183","https://openalex.org/W2104651652","https://openalex.org/W2116716943","https://openalex.org/W2127453322","https://openalex.org/W2133459682","https://openalex.org/W2147192413","https://openalex.org/W2154652894","https://openalex.org/W2163371195","https://openalex.org/W2164079290","https://openalex.org/W2250234233","https://openalex.org/W2250539671","https://openalex.org/W2252001469","https://openalex.org/W2344731424","https://openalex.org/W2525127255","https://openalex.org/W2534253848","https://openalex.org/W2564590796","https://openalex.org/W2604799547","https://openalex.org/W2606974598","https://openalex.org/W2729858395","https://openalex.org/W2739046565","https://openalex.org/W2741900445","https://openalex.org/W2806532810","https://openalex.org/W2888812214","https://openalex.org/W2896780650","https://openalex.org/W2909737760","https://openalex.org/W2914397182","https://openalex.org/W2962735233","https://openalex.org/W2963091658","https://openalex.org/W2963672599","https://openalex.org/W2963903950","https://openalex.org/W2964022663"],"related_works":["https://openalex.org/W2952523122","https://openalex.org/W2250842982","https://openalex.org/W2752442988","https://openalex.org/W1156568792","https://openalex.org/W260234060","https://openalex.org/W1601676518","https://openalex.org/W2803238307","https://openalex.org/W2107965514","https://openalex.org/W3110646668","https://openalex.org/W2162669657","https://openalex.org/W2785674851","https://openalex.org/W1598898694","https://openalex.org/W3111479136","https://openalex.org/W3040721418","https://openalex.org/W2044722804","https://openalex.org/W2339576758","https://openalex.org/W3118975918","https://openalex.org/W2251941743","https://openalex.org/W2898410387","https://openalex.org/W2251587334"],"abstract_inverted_index":{"Automatically":[0],"constructed":[1],"datasets":[2],"for":[3,74],"generating":[4],"text":[5,87],"from":[6,48,115,120],"semi-structured":[7,56],"data":[8,57,119],"(tables),":[9],"such":[10,24],"as":[11,25],"WikiBio":[12],"We":[13,39,90,102],"show":[14,29,77,103],"that":[15,78,104],"metrics":[16],"which":[17,45],"rely":[18],"solely":[19],"on":[20],"the":[21,49,55,95,110,118,121],"reference":[22,50,111],"texts,":[23],"BLEU":[26],"and":[27,51,62,93],"ROUGE,":[28],"poor":[30],"correlation":[31],"with":[32,81],"human":[33,68,82],"judgments":[34,83],"when":[35,109],"those":[36],"references":[37],"diverge.":[38],"propose":[40],"a":[41,65],"new":[42],"metric,":[43],"PAR-ENT,":[44],"aligns":[46],"n-grams":[47],"generated":[52],"texts":[53,112],"to":[54],"before":[58],"computing":[59],"their":[60],"precision":[61],"recall.":[63],"Through":[64],"large":[66],"scale":[67],"evaluation":[69,99],"study":[70],"of":[71],"table-to-text":[72],"models":[73],"WikiBio,":[75],"we":[76],"PARENT":[79,105],"correlates":[80],"better":[84],"than":[85],"existing":[86],"generation":[88],"metrics.":[89],"also":[91,107],"adapt":[92],"evaluate":[94],"information":[96],"extraction":[97],"based":[98],"proposed":[100],"in":[101],"is":[106],"applicable":[108],"are":[113],"elicited":[114],"humans":[116],"using":[117],"WebNLG":[122],"challenge.":[123]},"counts_by_year":[{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2025-10-10T00:00:00"}