{"id":"https://openalex.org/W2251610689","doi":"https://doi.org/10.3115/v1/w14-3336","title":"Results of the WMT14 Metrics Shared Task","display_name":"Results of the WMT14 Metrics Shared Task","publication_year":2014,"publication_date":"2014-01-01","ids":{"openalex":"https://openalex.org/W2251610689","doi":"https://doi.org/10.3115/v1/w14-3336","mag":"2251610689"},"language":"en","primary_location":{"id":"doi:10.3115/v1/w14-3336","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-3336","pdf_url":"https://aclanthology.org/W14-3336.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/W14-3336.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5032904654","display_name":"Matou\u0161 Mach\u00e1\u010bek","orcid":null},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":true,"raw_author_name":"Matous Machacek","raw_affiliation_strings":["Charles University in Prague"],"affiliations":[{"raw_affiliation_string":"Charles University in Prague","institution_ids":["https://openalex.org/I21250087"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5077746237","display_name":"Ond\u0159ej Bojar","orcid":"https://orcid.org/0000-0002-0606-0050"},"institutions":[{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ"],"is_corresponding":false,"raw_author_name":"Ondrej Bojar","raw_affiliation_strings":["Charles University in Prague"],"affiliations":[{"raw_affiliation_string":"Charles University in Prague","institution_ids":["https://openalex.org/I21250087"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5032904654"],"corresponding_institution_ids":["https://openalex.org/I21250087"],"apc_list":null,"apc_paid":null,"fwci":24.1018,"has_fulltext":true,"cited_by_count":109,"citation_normalized_percentile":{"value":0.99553958,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"293","last_page":"301"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/nist","display_name":"NIST","score":0.917546272277832},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7490556836128235},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.7380273342132568},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.727196455001831},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.7155998945236206},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6830485463142395},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6589995622634888},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4764668941497803},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4425009787082672},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.43116313219070435},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11451947689056396}],"concepts":[{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.917546272277832},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7490556836128235},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7380273342132568},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.727196455001831},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.7155998945236206},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6830485463142395},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6589995622634888},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4764668941497803},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4425009787082672},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.43116313219070435},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11451947689056396},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.3115/v1/w14-3336","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-3336","pdf_url":"https://aclanthology.org/W14-3336.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:-193319691194092630","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=-193319691194092630","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:-7125679027291272489","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=-7125679027291272489","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:-8469031730659020238","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=-8469031730659020238","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:pure.ed.ac.uk:publications/baba589b-9d25-4e48-8f4d-0bff25d7b274","is_oa":false,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/baba589b-9d25-4e48-8f4d-0bff25d7b274","pdf_url":null,"source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":""}],"best_oa_location":{"id":"doi:10.3115/v1/w14-3336","is_oa":true,"landing_page_url":"https://doi.org/10.3115/v1/w14-3336","pdf_url":"https://aclanthology.org/W14-3336.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Ninth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.4399999976158142}],"awards":[{"id":"https://openalex.org/G263766726","display_name":"Moses Open Source Evaluation and Support Co-ordination for OutReach and Exploitation","funder_award_id":"288487","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4358921014","display_name":null,"funder_award_id":"ICT-2011","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5483342299","display_name":null,"funder_award_id":"FP7-ICT","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G6544709687","display_name":null,"funder_award_id":"FP7-ICT-2011","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8614000523","display_name":null,"funder_award_id":"FP7-ICT-2011-7","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320338370","display_name":"FP7 Information and Communication Technologies","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2251610689.pdf","grobid_xml":"https://content.openalex.org/works/W2251610689.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W20811212","https://openalex.org/W56793832","https://openalex.org/W776270814","https://openalex.org/W1479933804","https://openalex.org/W2078861931","https://openalex.org/W2087735403","https://openalex.org/W2101105183","https://openalex.org/W2133459682","https://openalex.org/W2143177362","https://openalex.org/W2149327368","https://openalex.org/W2152311128","https://openalex.org/W2163726025","https://openalex.org/W2167897253","https://openalex.org/W2250188786","https://openalex.org/W2251476375","https://openalex.org/W2251610689","https://openalex.org/W2251994258","https://openalex.org/W2257408573","https://openalex.org/W2403821559","https://openalex.org/W2917452219","https://openalex.org/W6691655324"],"related_works":["https://openalex.org/W2158491338","https://openalex.org/W2807901368","https://openalex.org/W2133733652","https://openalex.org/W2072658171","https://openalex.org/W2606392311","https://openalex.org/W2320042380","https://openalex.org/W51364034","https://openalex.org/W2141938446","https://openalex.org/W2898073868","https://openalex.org/W4284663758"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"the":[3,6,19,22],"results":[4],"of":[5,14,21,34,48,68,83,88,103],"WMT14":[7,27,79],"Metrics":[8],"Shared":[9,28],"Task.":[10,30],"We":[11,31],"asked":[12],"participants":[13],"this":[15],"task":[16],"to":[17,43],"score":[18],"outputs":[20],"MT":[23],"systems":[24],"involved":[25],"in":[26,66,86,99],"Translation":[29],"collected":[32,62],"scores":[33,47,63,76],"23":[35],"metrics":[36,51],"from":[37],"12":[38],"research":[39],"groups.":[40],"In":[41],"addition":[42],"that":[44],"we":[45],"computed":[46],"6":[49],"standard":[50],"(BLEU,":[52],"NIST,":[53],"WER,":[54],"PER,":[55],"TER":[56],"and":[57,85],"CDER)":[58],"as":[59],"baselines.":[60],"The":[61],"were":[64],"evaluated":[65],"terms":[67,87],"system":[69],"level":[70,90],"correlation":[71,91],"(how":[72,92],"well":[73],"each":[74],"metric's":[75],"correlate":[77],"with":[78,97],"official":[80],"manual":[81],"ranking":[82],"systems)":[84],"segment":[89],"often":[93],"a":[94,104],"metric":[95],"agrees":[96],"humans":[98],"comparing":[100],"two":[101],"translations":[102],"particular":[105],"sentence).":[106],"DISCOTK-PARTY":[107],".970":[108],".010":[109],".921":[110],".024":[111],".862":[112],".":[113]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":6},{"year":2021,"cited_by_count":9},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":12},{"year":2018,"cited_by_count":8},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":19},{"year":2015,"cited_by_count":20},{"year":2014,"cited_by_count":10}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}