{"id":"https://openalex.org/W95780973","doi":null,"title":"Correlating automated and human assessments of machine translation quality","display_name":"Correlating automated and human assessments of machine translation quality","publication_year":2003,"publication_date":"2003-09-23","ids":{"openalex":"https://openalex.org/W95780973","mag":"95780973"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.694.8573","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.694.8573","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.mt-archive.info/MTS-2003-Coughlin.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5083291526","display_name":"Deborah A. Coughlin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Deborah A. Coughlin","raw_affiliation_strings":["Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5083291526"],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":5.8972,"has_fulltext":false,"cited_by_count":117,"citation_normalized_percentile":{"value":0.95953729,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9951000213623047,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0008999999845400453,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-translation-quality-assessment","display_name":"machine translation quality assessment","score":0.8830000162124634},{"id":"https://openalex.org/keywords/human-evaluation","display_name":"human evaluation","score":0.6740000247955322},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.925000011920929},{"id":"https://openalex.org/keywords/bleu","display_name":"BLEU","score":0.996999979019165},{"id":"https://openalex.org/keywords/nist","display_name":"NIST","score":0.9259999990463257},{"id":"https://openalex.org/keywords/assessment-methods","display_name":"assessment methods","score":0.3440000116825104}],"concepts":[{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.9538204669952393},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.7929251194000244},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7504622340202332},{"id":"https://openalex.org/C622187","wikidata":"https://www.wikidata.org/wiki/Q3500773","display_name":"BLEU","level":3,"score":0.6921678781509399},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6110324859619141},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5812926292419434},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5475741624832153},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.47186917066574097},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.47062376141548157},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.47058239579200745},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4528360068798065},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4232245087623596},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4213816225528717},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.41407907009124756},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10313469171524048},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.0},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.694.8573","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.694.8573","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.mt-archive.info/MTS-2003-Coughlin.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.81.8967","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.81.8967","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.amtaweb.org/summit/MTSummit/FinalPapers/41-Coughlin-final.pdf","raw_type":"text"},{"id":"mag:95780973","doi":null,"is_oa":false,"landing_page_url":"https://www.mt-archive.info/MTS-2003-Coughlin.pdf","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":6,"referenced_works":["https://openalex.org/W1529616844","https://openalex.org/W1973469546","https://openalex.org/W2078861931","https://openalex.org/W2101105183","https://openalex.org/W2162211144","https://openalex.org/W3213245459"],"related_works":["https://openalex.org/W2021618504","https://openalex.org/W2973065121","https://openalex.org/W6903579398","https://openalex.org/W3186081172","https://openalex.org/W2394620515","https://openalex.org/W2169279899","https://openalex.org/W2552929535","https://openalex.org/W2792202972","https://openalex.org/W2001810881","https://openalex.org/W7099313205"],"abstract_inverted_index":{"We":[0,29,87],"describe":[1],"a":[2],"large-scale":[3],"investigation":[4],"of":[5,11,33,37],"the":[6,16,27,31,42],"correlation":[7],"between":[8],"human":[9,35,70,85,91],"judgments":[10],"machine":[12,38],"translation":[13],"quality":[14],"and":[15,51,65],"automated":[17],"metrics":[18,49],"that":[19,89,109],"are":[20,55,93,110],"increasingly":[21],"used":[22],"to":[23,41,95],"drive":[24],"progress":[25],"in":[26],"field.":[28],"compare":[30],"results":[32],"124":[34],"evaluations":[36],"trans-lated":[39],"sentences":[40],"scores":[43,67,79],"generated":[44],"by":[45],"two":[46],"automatic":[47],"evaluation":[48],"(BLEU":[50],"NIST).":[52],"When":[53],"datasets":[54],"held":[56],"constant":[57],"or":[58,101],"file":[59],"size":[60],"is":[61],"sufficiently":[62],"large,":[63],"BLEU":[64],"NIST":[66],"closely":[68],"parallel":[69],"judgments.":[71],"Surprisingly,":[72],"this":[73],"was":[74],"true":[75],"even":[76],"though":[77],"these":[78],"were":[80],"calculated":[81],"using":[82],"just":[83],"one":[84],"reference.":[86],"suggest":[88],"when":[90],"evaluators":[92],"forced":[94],"make":[96],"decisions":[97],"without":[98],"sufficient":[99],"context":[100],"domain":[102],"expertise,":[103],"they":[104],"fall":[105],"back":[106],"on":[107],"strategies":[108],"not":[111],"unlike":[112],"determining":[113],"n-gram":[114],"precision.":[115],"1":[116]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":5},{"year":2014,"cited_by_count":5},{"year":2013,"cited_by_count":5},{"year":2012,"cited_by_count":6}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}