{"id":"https://openalex.org/W3036973371","doi":"https://doi.org/10.1609/aaai.v34i05.6521","title":"Learning to Compare for Better Training and Evaluation of Open Domain Natural Language Generation Models","display_name":"Learning to Compare for Better Training and Evaluation of Open Domain Natural Language Generation Models","alternate_titles":[],"publication_year":2020,"publication_date":"2020-04-03","ids":{"openalex":"https://openalex.org/W3036973371","doi":"https://doi.org/10.1609/aaai.v34i05.6521","mag":"3036973371"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v34i05.6521","doi":"https://doi.org/10.1609/aaai.v34i05.6521","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6521","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"parent_works":[],"lineage":["https://openalex.org/W3036973371"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1609/aaai.v34i05.6521","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5074508950","display_name":"Wangchunshu Zhou","orcid":"https://orcid.org/0000-0003-4668-3348","observed_orcids":["https://orcid.org/0000-0003-4668-3348"]},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wangchunshu Zhou","raw_affiliation_strings":["Beihang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5107234003","display_name":"Ke Xu","orcid":"https://orcid.org/0009-0007-5261-5620","observed_orcids":["https://orcid.org/0009-0007-5261-5620","https://orcid.org/0000-0001-5447-8008"]},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ke Xu","raw_affiliation_strings":["Beihang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beihang University","institution_ids":["https://openalex.org/I82880672"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I82880672"],"apc_list":null,"apc_paid":null,"fwci":2.1489,"has_fulltext":true,"cited_by_count":55,"citation_normalized_percentile":{"value":0.90029502,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"34","issue":"05","first_page":"9717","last_page":"9724"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5893999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.23549999296665192,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.15060000121593475,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automated-assessment","display_name":"automated assessment","score":0.7979999780654907},{"id":"https://openalex.org/keywords/bert-fine-tuning","display_name":"BERT fine-tuning","score":0.8579999804496765},{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.7680000066757202},{"id":"https://openalex.org/keywords/hyperparameter-optimization","display_name":"hyperparameter optimization","score":0.7829999923706055},{"id":"https://openalex.org/keywords/early-stopping","display_name":"early stopping","score":0.9879999756813049},{"id":"https://openalex.org/keywords/story-generation","display_name":"story generation","score":0.9649999737739563},{"id":"https://openalex.org/keywords/dialogue-response-generation","display_name":"dialogue response generation","score":0.8190000057220459},{"id":"https://openalex.org/keywords/human-evaluation-correlation","display_name":"human evaluation correlation","score":0.5220000147819519}],"concepts":[{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.9219499826431274},{"id":"https://openalex.org/C100279451","wikidata":"https://www.wikidata.org/wiki/Q372193","display_name":"Perplexity","level":3,"score":0.851318359375},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.823997974395752},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7061132192611694},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6653054356575012},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5978069305419922},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.5762492418289185},{"id":"https://openalex.org/C2993776861","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Open domain","level":3,"score":0.5469487309455872},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5183878540992737},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5065284967422485},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.49199482798576355},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.47017544507980347},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45152583718299866},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.41290417313575745},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.1766362190246582},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.12062016129493713},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v34i05.6521","doi":"https://doi.org/10.1609/aaai.v34i05.6521","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6521","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/6521","doi":null,"is_oa":true,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/6521","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/6521/6377","source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:arXiv.org:2002.05058","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2002.05058","pdf_url":"https://arxiv.org/pdf/2002.05058","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2002.05058","doi":"https://doi.org/10.48550/arxiv.2002.05058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2002.05058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v34i05.6521","doi":"https://doi.org/10.1609/aaai.v34i05.6521","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v34i05.6521","pdf_url":null,"source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3036973371.pdf","grobid_xml":"https://content.openalex.org/works/W3036973371.grobid-xml"},"referenced_works_count":31,"referenced_works":["https://openalex.org/W1516061453","https://openalex.org/W1518951372","https://openalex.org/W1597864774","https://openalex.org/W2064675550","https://openalex.org/W2099471712","https://openalex.org/W2101105183","https://openalex.org/W2123301721","https://openalex.org/W2130942839","https://openalex.org/W2131241448","https://openalex.org/W2153975459","https://openalex.org/W2154652894","https://openalex.org/W2296073425","https://openalex.org/W2581637843","https://openalex.org/W2583679610","https://openalex.org/W2606974598","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2741333084","https://openalex.org/W2761590056","https://openalex.org/W2886824226","https://openalex.org/W2896457183","https://openalex.org/W2903376039","https://openalex.org/W2936695845","https://openalex.org/W2950446064","https://openalex.org/W2963096510","https://openalex.org/W2963341956","https://openalex.org/W2963466651","https://openalex.org/W2963903950","https://openalex.org/W2970307457","https://openalex.org/W4320013936","https://openalex.org/W6898505805"],"related_works":["https://openalex.org/W6891696603","https://openalex.org/W1763968285","https://openalex.org/W4385572850","https://openalex.org/W3166758901","https://openalex.org/W4385571262","https://openalex.org/W3021107458","https://openalex.org/W4389519254","https://openalex.org/W3202653051","https://openalex.org/W2902236809","https://openalex.org/W4378876515"],"abstract_inverted_index":{"Automated":[0],"evaluation":[1,162],"of":[2,45,70,101,180],"open":[3],"domain":[4],"natural":[5,35,58],"language":[6,36,59],"generation":[7,37,141],"(NLG)":[8],"models":[9,38,72],"remains":[10],"a":[11,43,87,98,123],"challenge":[12],"and":[13,20,132,142],"widely":[14],"used":[15],"metrics":[16],"such":[17],"as":[18,122],"BLEU":[19],"Perplexity":[21],"can":[22,93],"be":[23,84,94],"misleading":[24],"in":[25,86,172],"some":[26],"cases.":[27],"In":[28,110],"our":[29,91,120,136,151],"paper,":[30],"we":[31,116],"propose":[32,64,117],"to":[33,41,55,65,83,105,112,118],"evaluate":[34,66,135],"by":[39,48],"learning":[40],"compare":[42],"pair":[44],"generated":[46],"sentences":[47],"fine-tuning":[49],"BERT,":[50],"which":[51,175],"has":[52],"been":[53],"shown":[54],"have":[56],"good":[57],"understanding":[60],"ability.":[61],"We":[62,134],"also":[63],"the":[67,166,178,181],"model-level":[68],"quality":[69],"NLG":[71],"with":[73,77,97,155,159,165],"sample-level":[74],"comparison":[75],"results":[76,148],"skill":[78],"rating":[79],"system.":[80],"While":[81],"able":[82],"trained":[85,114],"fully":[88],"self-supervised":[89],"fashion,":[90],"model":[92,121,152],"further":[95,176],"fine-tuned":[96],"little":[99],"amount":[100],"human":[102,108,156,173],"preference":[103,157],"annotation":[104],"better":[106,129,154,170],"imitate":[107],"judgment.":[109],"addition":[111],"evaluating":[113],"models,":[115],"apply":[119],"performance":[124,171],"indicator":[125],"during":[126],"training":[127],"for":[128],"hyperparameter":[130],"tuning":[131],"early-stopping.":[133],"approach":[137],"on":[138],"both":[139],"story":[140],"chit-chat":[143],"dialogue":[144],"response":[145],"generation.":[146],"Experimental":[147],"show":[149],"that":[150],"correlates":[153],"compared":[158],"previous":[160],"automated":[161],"approaches.":[163],"Training":[164],"proposed":[167,182],"metric":[168],"yields":[169],"evaluation,":[174],"demonstrates":[177],"effectiveness":[179],"model.":[183]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3036973371","counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":7},{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":19},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":6}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2020-06-25T00:00:00"}
