{"id":"https://openalex.org/W3159892921","doi":"https://doi.org/10.1162/tacl_a_00437","title":"Experts, Errors, and Context: A Large-Scale Study of Human Evaluation for Machine Translation","display_name":"Experts, Errors, and Context: A Large-Scale Study of Human Evaluation for Machine Translation","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3159892921","doi":"https://doi.org/10.1162/tacl_a_00437","mag":"3159892921"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00437","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00437","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00437/1979261/tacl_a_00437.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00437/1979261/tacl_a_00437.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068438716","display_name":"Markus Freitag","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Markus Freitag","raw_affiliation_strings":["Google Research. freitag@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. freitag@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108540870","display_name":"George Foster","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"George Foster","raw_affiliation_strings":["Google Research. fosterg@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. fosterg@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065912572","display_name":"David Grangier","orcid":"https://orcid.org/0000-0002-8847-9532"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"David Grangier","raw_affiliation_strings":["Google Research. grangier@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. grangier@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054443211","display_name":"Viresh Ratnakar","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Viresh Ratnakar","raw_affiliation_strings":["Google Research. vratnakar@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. vratnakar@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065863466","display_name":"Qijun Tan","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Qijun Tan","raw_affiliation_strings":["Google Research. qijuntan@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. qijuntan@google.com","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018646392","display_name":"Wolfgang Macherey","orcid":null},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Wolfgang Macherey","raw_affiliation_strings":["Google Research. wmach@google.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research. wmach@google.com","institution_ids":["https://openalex.org/I1291425158"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5018646392","https://openalex.org/A5054443211","https://openalex.org/A5065863466","https://openalex.org/A5065912572","https://openalex.org/A5068438716","https://openalex.org/A5108540870"],"corresponding_institution_ids":["https://openalex.org/I1291425158"],"apc_list":null,"apc_paid":null,"fwci":23.5781,"has_fulltext":false,"cited_by_count":249,"citation_normalized_percentile":{"value":0.99648869,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"9","issue":null,"first_page":"1460","last_page":"1474"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9944000244140625,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8612173795700073},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.7737400531768799},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7006610631942749},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6780911087989807},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.6247580051422119},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6196490526199341},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5766662955284119},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5274955034255981},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5186804533004761},{"id":"https://openalex.org/keywords/human\u2013machine-system","display_name":"Human\u2013machine system","score":0.48623910546302795},{"id":"https://openalex.org/keywords/evaluation-of-machine-translation","display_name":"Evaluation of machine translation","score":0.4508107900619507},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4300192594528198},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4184757471084595},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3766274154186249},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.33234915137290955},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.33167126774787903},{"id":"https://openalex.org/keywords/machine-translation-software-usability","display_name":"Machine translation software usability","score":0.08526742458343506},{"id":"https://openalex.org/keywords/example-based-machine-translation","display_name":"Example-based machine translation","score":0.0694572925567627},{"id":"https://openalex.org/keywords/systems-engineering","display_name":"Systems engineering","score":0.06311231851577759}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8612173795700073},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.7737400531768799},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7006610631942749},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6780911087989807},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.6247580051422119},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6196490526199341},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5766662955284119},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5274955034255981},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5186804533004761},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.48623910546302795},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.4508107900619507},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4300192594528198},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4184757471084595},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3766274154186249},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.33234915137290955},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.33167126774787903},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.08526742458343506},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.0694572925567627},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.06311231851577759},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/tacl_a_00437","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00437","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00437/1979261/tacl_a_00437.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2104.14478","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2104.14478","pdf_url":"https://arxiv.org/pdf/2104.14478","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:40f406ebbb294289a01d49fc645aacb2","is_oa":false,"landing_page_url":"https://doaj.org/article/40f406ebbb294289a01d49fc645aacb2","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 9, Pp 1460-1474 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00437","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00437","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00437/1979261/tacl_a_00437.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W1504725289","https://openalex.org/W1654441844","https://openalex.org/W1985900877","https://openalex.org/W2148365102","https://openalex.org/W2148504207","https://openalex.org/W2152311128","https://openalex.org/W2250342921","https://openalex.org/W2252166243","https://openalex.org/W2260677151","https://openalex.org/W2512924740","https://openalex.org/W2524643686","https://openalex.org/W2531638282","https://openalex.org/W2575371842","https://openalex.org/W2738658885","https://openalex.org/W2741595787","https://openalex.org/W2750811310","https://openalex.org/W2760656271","https://openalex.org/W2787568264","https://openalex.org/W2794365787","https://openalex.org/W2795152750","https://openalex.org/W2888159079","https://openalex.org/W2890626829","https://openalex.org/W2899689163","https://openalex.org/W2922158773","https://openalex.org/W2950617329","https://openalex.org/W2952103439","https://openalex.org/W2970038984","https://openalex.org/W2971043182","https://openalex.org/W3013197936","https://openalex.org/W3037252522","https://openalex.org/W3081828086","https://openalex.org/W3103424038","https://openalex.org/W3103450644","https://openalex.org/W3104976898","https://openalex.org/W3105214104","https://openalex.org/W3106146701","https://openalex.org/W3113362544","https://openalex.org/W3114770456","https://openalex.org/W3119878165","https://openalex.org/W3120929527","https://openalex.org/W6682058439","https://openalex.org/W6691459612","https://openalex.org/W6732194750","https://openalex.org/W6743573882","https://openalex.org/W6749740488","https://openalex.org/W6749806535","https://openalex.org/W6763585265","https://openalex.org/W6764464928","https://openalex.org/W6767250937","https://openalex.org/W6777499153","https://openalex.org/W6779514328","https://openalex.org/W6779614783","https://openalex.org/W6788156848"],"related_works":["https://openalex.org/W3151736118","https://openalex.org/W3097971125","https://openalex.org/W2962780935","https://openalex.org/W4387490002","https://openalex.org/W2169279899","https://openalex.org/W2026303297","https://openalex.org/W2532807140","https://openalex.org/W3196981096","https://openalex.org/W2395826315","https://openalex.org/W2152633844"],"abstract_inverted_index":{"Abstract":[0],"Human":[1],"evaluation":[2,20,54],"of":[3,82,121],"modern":[4],"high-quality":[5],"machine":[6,140],"translation":[7],"systems":[8,84,123],"is":[9,15],"a":[10,40,46,117,134],"difficult":[11],"problem,":[12],"and":[13],"there":[14,28],"increasing":[16],"evidence":[17],"that":[18,146],"inadequate":[19],"procedures":[21],"can":[22,153],"lead":[23],"to":[24,77,103],"erroneous":[25],"conclusions.":[26],"While":[27],"has":[29],"been":[30],"considerable":[31],"research":[32,75],"on":[33,62,150],"human":[34,138,155],"evaluation,":[35],"the":[36,63,72,80,86,109,125,129],"field":[37],"still":[38],"lacks":[39],"commonly":[41],"accepted":[42],"standard":[43],"procedure.":[44],"As":[45],"step":[47],"toward":[48],"this":[49],"goal,":[50],"we":[51,143],"propose":[52],"an":[53],"methodology":[55],"grounded":[56],"in":[57,91],"explicit":[58],"error":[59],"analysis,":[60],"based":[61,149],"Multidimensional":[64],"Quality":[65],"Metrics":[66],"(MQM)":[67],"framework.":[68],"We":[69,107,158],"carry":[70],"out":[71],"largest":[73],"MQM":[74],"study":[76],"date,":[78],"scoring":[79],"outputs":[81],"top":[83],"from":[85,124],"WMT":[87,130],"2020":[88],"shared":[89],"task":[90],"two":[92],"language":[93],"pairs":[94],"using":[95],"annotations":[96],"provided":[97],"by":[98,128],"professional":[99],"translators":[100],"with":[101],"access":[102],"full":[104],"document":[105],"context.":[106],"analyze":[108],"resulting":[110],"data":[111],"extensively,":[112],"finding":[113],"among":[114],"other":[115],"results":[116],"substantially":[118],"different":[119],"ranking":[120],"evaluated":[122],"one":[126],"established":[127],"crowd":[131,156],"workers,":[132],"exhibiting":[133],"clear":[135],"preference":[136],"for":[137,164],"over":[139],"output.":[141],"Surprisingly,":[142],"also":[144],"find":[145],"automatic":[147],"metrics":[148],"pre-trained":[151],"embeddings":[152],"outperform":[154],"workers.":[157],"make":[159],"our":[160],"corpus":[161],"publicly":[162],"available":[163],"further":[165],"research.":[166]},"counts_by_year":[{"year":2026,"cited_by_count":22},{"year":2025,"cited_by_count":46},{"year":2024,"cited_by_count":35},{"year":2023,"cited_by_count":74},{"year":2022,"cited_by_count":46},{"year":2021,"cited_by_count":26}],"updated_date":"2026-08-06T08:24:18.245995","created_date":"2021-05-10T00:00:00"}