{"id":"https://openalex.org/W2949611393","doi":"https://doi.org/10.18653/v1/p19-1427","title":"Beyond BLEU: Training Neural Machine Translation with Semantic Similarity","display_name":"Beyond BLEU: Training Neural Machine Translation with Semantic Similarity","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2949611393","doi":"https://doi.org/10.18653/v1/p19-1427","mag":"2949611393"},"language":"en","primary_location":{"id":"doi:10.18653/v1/p19-1427","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1427","pdf_url":"https://www.aclweb.org/anthology/P19-1427.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/P19-1427.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002499277","display_name":"John Wieting","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Wieting","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, 15213, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, 15213, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017455302","display_name":"Taylor Berg-Kirkpatrick","orcid":"https://orcid.org/0000-0002-1283-4075"},"institutions":[{"id":"https://openalex.org/I36258959","display_name":"University of California San Diego","ror":"https://ror.org/0168r3w48","country_code":"US","type":"education","lineage":["https://openalex.org/I36258959"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Taylor Berg-Kirkpatrick","raw_affiliation_strings":["University of California San Diego, San Diego, CA, 92093, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of California San Diego, San Diego, CA, 92093, USA","institution_ids":["https://openalex.org/I36258959"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081022650","display_name":"Kevin Gimpel","orcid":null},"institutions":[{"id":"https://openalex.org/I160992636","display_name":"Toyota Technological Institute at Chicago","ror":"https://ror.org/02sn5gb64","country_code":"US","type":"education","lineage":["https://openalex.org/I160992636"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Gimpel","raw_affiliation_strings":["Toyota Technological Institute at Chicago, Chicago, IL, 60637, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Toyota Technological Institute at Chicago, Chicago, IL, 60637, USA","institution_ids":["https://openalex.org/I160992636"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068811427","display_name":"Graham Neubig","orcid":"https://orcid.org/0000-0002-2072-3789"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Graham Neubig","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, 15213, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, 15213, USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":136,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4344","last_page":"4355"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bleu","display_name":"BLEU","score":0.9555546045303345},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8630592823028564},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.8535414338111877},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7570291757583618},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.7118549346923828},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.6892721056938171},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.633054792881012},{"id":"https://openalex.org/keywords/evaluation-of-machine-translation","display_name":"Evaluation of machine translation","score":0.623699426651001},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5807704925537109},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5183283090591431},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4667876958847046},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"Semantic similarity","score":0.45418137311935425},{"id":"https://openalex.org/keywords/example-based-machine-translation","display_name":"Example-based machine translation","score":0.2028137743473053},{"id":"https://openalex.org/keywords/machine-translation-software-usability","display_name":"Machine translation software usability","score":0.0620291531085968}],"concepts":[{"id":"https://openalex.org/C622187","wikidata":"https://www.wikidata.org/wiki/Q3500773","display_name":"BLEU","level":3,"score":0.9555546045303345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8630592823028564},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.8535414338111877},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7570291757583618},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.7118549346923828},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.6892721056938171},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.633054792881012},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.623699426651001},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5807704925537109},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5183283090591431},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4667876958847046},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.45418137311935425},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.2028137743473053},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.0620291531085968},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/p19-1427","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1427","pdf_url":"https://www.aclweb.org/anthology/P19-1427.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/p19-1427","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/p19-1427","pdf_url":"https://www.aclweb.org/anthology/P19-1427.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2949611393.pdf","grobid_xml":"https://content.openalex.org/works/W2949611393.grobid-xml"},"referenced_works_count":49,"referenced_works":["https://openalex.org/W91928571","https://openalex.org/W104184427","https://openalex.org/W1815076433","https://openalex.org/W2087735403","https://openalex.org/W2099607809","https://openalex.org/W2101105183","https://openalex.org/W2110310640","https://openalex.org/W2126400076","https://openalex.org/W2131744502","https://openalex.org/W2133458109","https://openalex.org/W2133459682","https://openalex.org/W2133564696","https://openalex.org/W2137143056","https://openalex.org/W2142112143","https://openalex.org/W2146574666","https://openalex.org/W2152180407","https://openalex.org/W2176263492","https://openalex.org/W2180952760","https://openalex.org/W2183341477","https://openalex.org/W2251047310","https://openalex.org/W2251473834","https://openalex.org/W2251861449","https://openalex.org/W2252028648","https://openalex.org/W2462305634","https://openalex.org/W2513167989","https://openalex.org/W2581377246","https://openalex.org/W2601733547","https://openalex.org/W2605035112","https://openalex.org/W2613904329","https://openalex.org/W2760656271","https://openalex.org/W2895810819","https://openalex.org/W2903193068","https://openalex.org/W2922349260","https://openalex.org/W2949547296","https://openalex.org/W2951535825","https://openalex.org/W2962784628","https://openalex.org/W2963248296","https://openalex.org/W2963463964","https://openalex.org/W2963499246","https://openalex.org/W2963620441","https://openalex.org/W2963804993","https://openalex.org/W2963899155","https://openalex.org/W2963918774","https://openalex.org/W2964190861","https://openalex.org/W2964212550","https://openalex.org/W2964265128","https://openalex.org/W2964308564","https://openalex.org/W4295803813","https://openalex.org/W4297798436"],"related_works":["https://openalex.org/W3151736118","https://openalex.org/W2395641992","https://openalex.org/W4280571180","https://openalex.org/W2099607809","https://openalex.org/W3021126373","https://openalex.org/W92588874","https://openalex.org/W2518299649","https://openalex.org/W2807475932","https://openalex.org/W2140343992","https://openalex.org/W2903057408"],"abstract_inverted_index":{"While":[0],"most":[1],"neural":[2],"machine":[3],"translation":[4,33],"(NMT)":[5],"systems":[6,20,81],"are":[7],"still":[8],"trained":[9],"using":[10],"maximum":[11],"likelihood":[12],"estimation,":[13],"recent":[14,86],"work":[15,87],"has":[16,39,48],"demonstrated":[17],"that":[18,82,102,123,131],"optimizing":[19,79],"to":[21,98,141],"directly":[22],"improve":[23,31],"evaluation":[24],"metrics":[25],"such":[26],"as":[27,112],"BLEU":[28,38],"can":[29,57],"substantially":[30],"final":[32],"accuracy.":[34],"However,":[35],"training":[36,103],"with":[37,104],"some":[40],"limitations:":[41],"it":[42,47,56],"doesn't":[43],"assign":[44],"partial":[45,144],"credit,":[46],"a":[49],"limited":[50],"range":[51],"of":[52],"output":[53],"values,":[54],"and":[55,100,118,121,146],"penalize":[58],"semantically":[59],"correct":[60],"hypotheses":[61],"if":[62],"they":[63],"differ":[64],"lexically":[65],"from":[66],"the":[67,124,135],"reference.":[68],"In":[69],"this":[70,132],"paper,":[71],"we":[72],"introduce":[73],"an":[74],"alternative":[75],"reward":[76],"function":[77],"for":[78],"NMT":[80],"is":[83,133,138],"based":[84],"on":[85,93],"in":[88,109,150],"semantic":[89,116],"similarity.":[90],"We":[91],"evaluate":[92],"four":[94],"disparate":[95],"languages":[96],"translated":[97],"English,":[99],"find":[101],"our":[105],"proposed":[106,136],"metric":[107,137],"results":[108],"better":[110],"translations":[111],"evaluated":[113],"by":[114],"BLEU,":[115],"similarity,":[117],"human":[119],"evaluation,":[120],"also":[122],"optimization":[125],"procedure":[126],"converges":[127],"faster.":[128],"Analysis":[129],"suggests":[130],"because":[134],"more":[139,148],"conducive":[140],"optimization,":[142],"assigning":[143],"credit":[145],"providing":[147],"diversity":[149],"scores":[151],"than":[152],"BLEU.":[153],"1":[154]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":12},{"year":2023,"cited_by_count":38},{"year":2022,"cited_by_count":27},{"year":2021,"cited_by_count":19},{"year":2020,"cited_by_count":16},{"year":2019,"cited_by_count":4}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}