{"id":"https://openalex.org/W2552556052","doi":null,"title":"The Limits of N-Gram Translation Evaluation Metrics","display_name":"The Limits of N-Gram Translation Evaluation Metrics","publication_year":2003,"publication_date":"2003-09-23","ids":{"openalex":"https://openalex.org/W2552556052","mag":"2552556052"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.469.900","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.469.900","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.mt-archive.info/MTS-2003-Culy.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5108346880","display_name":"Christopher Culy","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I26274970","display_name":"FX Palo Alto Laboratory","ror":"https://ror.org/05ef4zt88","country_code":"US","type":"facility","lineage":["https://openalex.org/I26274970"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher Culy","raw_affiliation_strings":["[FX Palo Alto Laboratory]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"[FX Palo Alto Laboratory]","institution_ids":["https://openalex.org/I26274970"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5048581550","display_name":"Susanne Z. Riehemann","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1298353152","display_name":"SRI International","ror":"https://ror.org/05s570m15","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I1298353152"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Susanne Z. Riehemann","raw_affiliation_strings":["SRI-International"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SRI-International","institution_ids":["https://openalex.org/I1298353152"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.8158,"has_fulltext":false,"cited_by_count":49,"citation_normalized_percentile":{"value":0.93762202,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":87,"max":98},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bleu","display_name":"BLEU","score":0.9909999966621399},{"id":"https://openalex.org/keywords/machine-translation-quality-assessment","display_name":"machine translation quality assessment","score":0.9300000071525574},{"id":"https://openalex.org/keywords/human-vs-machine-translation-comparison","display_name":"human vs machine translation comparison","score":0.6859999895095825},{"id":"https://openalex.org/keywords/bible-translation","display_name":"Bible translation","score":0.6039999723434448},{"id":"https://openalex.org/keywords/tom-sawyer","display_name":"Tom Sawyer","score":0.8360000252723694},{"id":"https://openalex.org/keywords/literary-translation","display_name":"literary translation","score":0.7409999966621399},{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.6190000176429749}],"concepts":[{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.8019464015960693},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.7497283816337585},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6735939383506775},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6040438413619995},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.5999196767807007},{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.5880685448646545},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5455477237701416},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5308322310447693},{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.48663195967674255},{"id":"https://openalex.org/C622187","wikidata":"https://www.wikidata.org/wiki/Q3500773","display_name":"BLEU","level":3,"score":0.4568816125392914},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.4496803283691406},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.3105314373970032},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3024856746196747},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.2734423279762268},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.08354184031486511},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0813729465007782},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.07623690366744995},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.469.900","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.469.900","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.mt-archive.info/MTS-2003-Culy.pdf","raw_type":"text"},{"id":"mag:2552556052","doi":null,"is_oa":false,"landing_page_url":"https://aclanthology.org/2003.mtsummit-papers.10/","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":4,"referenced_works":["https://openalex.org/W645795883","https://openalex.org/W1588612820","https://openalex.org/W2016630033","https://openalex.org/W2101105183"],"related_works":["https://openalex.org/W2394620515","https://openalex.org/W95780973","https://openalex.org/W2132122285","https://openalex.org/W1992344231","https://openalex.org/W7096014429","https://openalex.org/W2078861931","https://openalex.org/W3198028687","https://openalex.org/W2250484373","https://openalex.org/W2250214155","https://openalex.org/W4416420673"],"abstract_inverted_index":{"N-gram":[0],"measures":[1,109],"of":[2,34,43,113,124],"translation":[3,115],"quality,":[4,116],"such":[5],"as":[6,122],"BLEU":[7],"and":[8,49],"the":[9,32,47,125],"related":[10],"NIST":[11],"metric,":[12],"are":[13,23,119],"becoming":[14],"increasingly":[15],"important":[16],"in":[17,106,110],"machine":[18,59,79,114],"translation,":[19],"yet":[20],"their":[21],"behaviors":[22],"not":[24],"fully":[25],"understood.":[26],"In":[27,66],"this":[28],"paper":[29],"we":[30],"examine":[31],"performance":[33],"these":[35],"metrics":[36],"on":[37],"profes-sional":[38],"human":[39,64,76],"translations":[40,60,77,80,89],"into":[41],"German":[42],"two":[44,83],"literary":[45],"genres,":[46],"Bible":[48],"Tom":[50],"Sawyer.":[51],"The":[52],"most":[53],"surprising":[54],"result":[55],"is":[56,91],"that":[57,100],"some":[58,62,74],"outscore":[61],"professional":[63],"translations.":[65],"addition,":[67],"it":[68,90],"can":[69],"be":[70,104],"difficult":[71],"to":[72,98],"distinguish":[73],"other":[75],"from":[78],"with":[81,86],"only":[82],"reference":[84,88],"translations;":[85],"four":[87],"much":[92,101],"easier.":[93],"Our":[94],"results":[95],"lead":[96],"us":[97],"conclude":[99],"care":[102],"must":[103],"taken":[105],"using":[107],"n-gram":[108],"formal":[111],"evaluations":[112],"though":[117],"they":[118],"still":[120],"valuable":[121],"part":[123],"iterative":[126],"development":[127],"cycle.":[128]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":2},{"year":2017,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":1},{"year":2012,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-11-30T00:00:00"}