{"id":"https://openalex.org/W2973065121","doi":"https://doi.org/10.1017/s1351324919000469","title":"How to evaluate machine translation: A review of automated and human metrics","display_name":"How to evaluate machine translation: A review of automated and human metrics","publication_year":2019,"publication_date":"2019-09-11","ids":{"openalex":"https://openalex.org/W2973065121","doi":"https://doi.org/10.1017/s1351324919000469","mag":"2973065121"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324919000469","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324919000469","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031492370","display_name":"Eirini Chatzikoumi","orcid":"https://orcid.org/0000-0002-4628-6089"},"institutions":[{"id":"https://openalex.org/I130474213","display_name":"Pontificia Universidad Cat\u00f3lica de Valpara\u00edso","ror":"https://ror.org/02cafbr77","country_code":"CL","type":"education","lineage":["https://openalex.org/I130474213"]}],"countries":["CL"],"is_corresponding":true,"raw_author_name":"Eirini Chatzikoumi","raw_affiliation_strings":["Instituto de Literatura y Ciencias del Lenguaje, Pontificia Universidad Cat\u00f3lica de Valpara\u00edso, Av. El Bosque 1290, Vi\u00f1a del Mar, Chile"],"raw_orcid":"https://orcid.org/0000-0002-4628-6089","affiliations":[{"raw_affiliation_string":"Instituto de Literatura y Ciencias del Lenguaje, Pontificia Universidad Cat\u00f3lica de Valpara\u00edso, Av. El Bosque 1290, Vi\u00f1a del Mar, Chile","institution_ids":["https://openalex.org/I130474213"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5031492370"],"corresponding_institution_ids":["https://openalex.org/I130474213"],"apc_list":null,"apc_paid":null,"fwci":3.7737,"has_fulltext":false,"cited_by_count":98,"citation_normalized_percentile":{"value":0.94734502,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"26","issue":"2","first_page":"137","last_page":"161"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9840999841690063,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8455535173416138},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.5955047011375427},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.5518067479133606},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5232933163642883},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5216686725616455},{"id":"https://openalex.org/keywords/evaluation-of-machine-translation","display_name":"Evaluation of machine translation","score":0.513615071773529},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4956001043319702},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.48572951555252075},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4715331196784973},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4174238443374634},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.32191550731658936},{"id":"https://openalex.org/keywords/machine-translation-software-usability","display_name":"Machine translation software usability","score":0.10260626673698425}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8455535173416138},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.5955047011375427},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.5518067479133606},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5232933163642883},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5216686725616455},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.513615071773529},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4956001043319702},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.48572951555252075},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4715331196784973},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4174238443374634},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.32191550731658936},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.10260626673698425},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s1351324919000469","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s1351324919000469","pdf_url":null,"source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.5899999737739563,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":81,"referenced_works":["https://openalex.org/W20811212","https://openalex.org/W105964407","https://openalex.org/W245306944","https://openalex.org/W568173204","https://openalex.org/W652108813","https://openalex.org/W1482044060","https://openalex.org/W1516184288","https://openalex.org/W1517640241","https://openalex.org/W1529616844","https://openalex.org/W1572036451","https://openalex.org/W1579903274","https://openalex.org/W1647671624","https://openalex.org/W1916559533","https://openalex.org/W1996456980","https://openalex.org/W2001496424","https://openalex.org/W2001771035","https://openalex.org/W2021618504","https://openalex.org/W2026828282","https://openalex.org/W2037831268","https://openalex.org/W2051840895","https://openalex.org/W2060833990","https://openalex.org/W2064354777","https://openalex.org/W2088781183","https://openalex.org/W2098507980","https://openalex.org/W2101105183","https://openalex.org/W2108325777","https://openalex.org/W2113343614","https://openalex.org/W2126118742","https://openalex.org/W2137901323","https://openalex.org/W2141848945","https://openalex.org/W2143803796","https://openalex.org/W2145482038","https://openalex.org/W2147192413","https://openalex.org/W2148437670","https://openalex.org/W2152311128","https://openalex.org/W2160001241","https://openalex.org/W2160553030","https://openalex.org/W2164984707","https://openalex.org/W2250342921","https://openalex.org/W2251251208","https://openalex.org/W2260677151","https://openalex.org/W2292605867","https://openalex.org/W2396065825","https://openalex.org/W2471690557","https://openalex.org/W2483327705","https://openalex.org/W2524643686","https://openalex.org/W2525778437","https://openalex.org/W2550419310","https://openalex.org/W2558224492","https://openalex.org/W2579599705","https://openalex.org/W2588998509","https://openalex.org/W2595813107","https://openalex.org/W2607303097","https://openalex.org/W2613194865","https://openalex.org/W2613253298","https://openalex.org/W2621975677","https://openalex.org/W2625092622","https://openalex.org/W2738658885","https://openalex.org/W2759083144","https://openalex.org/W2771472444","https://openalex.org/W2787568264","https://openalex.org/W2792932412","https://openalex.org/W2794365787","https://openalex.org/W2799915114","https://openalex.org/W2801499199","https://openalex.org/W2888159079","https://openalex.org/W2899750749","https://openalex.org/W2922158773","https://openalex.org/W2922439213","https://openalex.org/W2949888546","https://openalex.org/W2950617329","https://openalex.org/W2964130895","https://openalex.org/W2964199361","https://openalex.org/W3196862065","https://openalex.org/W4230547814","https://openalex.org/W4234188104","https://openalex.org/W4285719527","https://openalex.org/W6636915900","https://openalex.org/W6679436768","https://openalex.org/W6898505805","https://openalex.org/W6983553224"],"related_works":["https://openalex.org/W3151736118","https://openalex.org/W3097971125","https://openalex.org/W2962780935","https://openalex.org/W2026303297","https://openalex.org/W4387490002","https://openalex.org/W2250213760","https://openalex.org/W2110168585","https://openalex.org/W3107474891","https://openalex.org/W2775554247","https://openalex.org/W2587014613"],"abstract_inverted_index":{"Abstract":[0],"This":[1,44],"article":[2,45,208],"presents":[3],"the":[4,16,25,40,51,59,65,68,114,137,160,168,188,216,220,224,239,246,251,260],"most":[5,247],"up-to-date,":[6],"influential":[7],"automated,":[8],"semiautomated":[9],"and":[10,23,64,100,176,180,197,203],"human":[11,62,118,243,261],"metrics":[12,109,262],"used":[13,94],"to":[14,56,61,67,113,238,268],"evaluate":[15,278],"quality":[17,88,98,252],"of":[18,42,116,219,226,250,253],"machine":[19,73],"translation":[20,74],"(MT)":[21],"output":[22],"provides":[24],"necessary":[26],"background":[27],"for":[28,39,97,212,234,273],"MT":[29,279],"evaluation":[30,102,108,125,157,165,170,190,213],"projects.":[31,214],"Evaluation":[32],"is,":[33],"as":[34,95,128,135,174,194],"repeatedly":[35],"admitted,":[36],"highly":[37],"relevant":[38],"improvement":[41],"MT.":[43],"is":[46,54,136,257],"divided":[47],"into":[48],"three":[49],"parts:":[50],"first":[52,80],"one":[53],"dedicated":[55],"automated":[57],"metrics;":[58,63,85],"second,":[60],"last,":[66],"challenges":[69],"posed":[70],"by":[71],"neural":[72],"(NMT)":[75],"regarding":[76],"its":[77,235],"evaluation.":[78],"The":[79,142],"part":[81],"includes":[82],"reference":[83],"translation\u2013based":[84],"confidence":[86],"or":[87,130,133],"estimation":[89],"(QE)":[90],"metrics,":[91,223],"which":[92],"are":[93,110,145,154,162,184,199,228,245],"alternatives":[96],"assessment;":[99],"diagnostic":[101],"based":[103,146],"on":[104,147,259],"linguistic":[105],"checkpoints.":[106],"Human":[107],"classified":[111],"according":[112,237],"criterion":[115],"whether":[117],"judges":[119,249],"directly":[120,148],"express":[121],"a":[122,209,254,265],"so-called":[123],"subjective":[124],"judgment,":[126],"such":[127,173,193],"\u2018good\u2019":[129],"\u2018better":[131],"than\u2019,":[132],"not,":[134],"case":[138],"in":[139,187],"error":[140,195],"classification.":[141],"former":[143],"methods":[144],"expressed":[149],"judgment":[150],"(DEJ);":[151],"therefore,":[152],"they":[153],"called":[155,163],"\u2018DEJ-based":[156],"methods\u2019,":[158],"while":[159],"latter":[161],"\u2018non-DEJ-based":[164],"methods\u2019.":[166],"In":[167],"DEJ-based":[169],"section,":[171,191],"tasks":[172,192],"fluency":[175],"adequacy":[177],"annotation,":[178],"ranking":[179],"direct":[181],"assessment":[182],"(DA)":[183],"presented,":[185],"whereas":[186],"non-DEJ-based":[189],"classification":[196],"postediting":[198],"detailed,":[200],"with":[201,232],"definitions":[202],"guidelines,":[204],"thus":[205],"rendering":[206],"this":[207],"useful":[210,270],"guide":[211],"Following":[215],"detailed":[217],"presentation":[218],"previously":[221],"mentioned":[222],"specificities":[225],"NMT":[227],"set":[229],"forth":[230],"along":[231],"suggestions":[233],"evaluation,":[236],"latest":[240],"studies.":[241],"As":[242],"translators":[244],"adequate":[248],"translation,":[255],"emphasis":[256],"placed":[258],"seen":[263],"from":[264],"translator-judge":[266],"perspective":[267],"provide":[269],"methodology":[271],"tools":[272],"interdisciplinary":[274],"research":[275],"groups":[276],"that":[277],"systems.":[280]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":20},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":29},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":15},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":1}],"updated_date":"2026-05-15T08:27:34.491423","created_date":"2025-10-10T00:00:00"}