{"id":"https://openalex.org/W2508316494","doi":"https://doi.org/10.18653/v1/w16-2302","title":"Results of the WMT16 Metrics Shared Task","display_name":"Results of the WMT16 Metrics Shared Task","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2508316494","doi":"https://doi.org/10.18653/v1/w16-2302","mag":"2508316494"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w16-2302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2302","pdf_url":"https://www.aclweb.org/anthology/W16-2302.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First Conference on Machine Translation: Volume 2,\n          Shared Task Papers","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W16-2302.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077746237","display_name":"Ond\u0159ej Bojar","orcid":"https://orcid.org/0000-0002-0606-0050"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]},{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ","IE","NL"],"is_corresponding":true,"raw_author_name":"Ond\u0159ej Bojar","raw_affiliation_strings":["MFF \u00daFAL","Dublin City Univ. ADAPT","Charles Univ. in Prague","Univ. of Amsterdam ILLC"],"affiliations":[{"raw_affiliation_string":"MFF \u00daFAL","institution_ids":[]},{"raw_affiliation_string":"Dublin City Univ. ADAPT","institution_ids":["https://openalex.org/I42934936"]},{"raw_affiliation_string":"Charles Univ. in Prague","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Univ. of Amsterdam ILLC","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020500064","display_name":"Yvette Graham","orcid":"https://orcid.org/0000-0001-6741-4855"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]},{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["CZ","IE","NL"],"is_corresponding":false,"raw_author_name":"Yvette Graham","raw_affiliation_strings":["MFF \u00daFAL","Charles Univ. in Prague","Univ. of Amsterdam ILLC","Dublin City Univ. ADAPT"],"affiliations":[{"raw_affiliation_string":"MFF \u00daFAL","institution_ids":[]},{"raw_affiliation_string":"Charles Univ. in Prague","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"Univ. of Amsterdam ILLC","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"Dublin City Univ. ADAPT","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108625478","display_name":"Amir Kamran","orcid":null},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]},{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ","IE","NL"],"is_corresponding":false,"raw_author_name":"Amir Kamran","raw_affiliation_strings":["Univ. of Amsterdam ILLC","Charles Univ. in Prague","MFF \u00daFAL","Dublin City Univ. ADAPT"],"affiliations":[{"raw_affiliation_string":"Univ. of Amsterdam ILLC","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"Charles Univ. in Prague","institution_ids":["https://openalex.org/I21250087"]},{"raw_affiliation_string":"MFF \u00daFAL","institution_ids":[]},{"raw_affiliation_string":"Dublin City Univ. ADAPT","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074399519","display_name":"Milo\u0161 Stanojevi\u0107","orcid":"https://orcid.org/0000-0001-8100-9864"},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]},{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]},{"id":"https://openalex.org/I21250087","display_name":"Charles University","ror":"https://ror.org/024d6js02","country_code":"CZ","type":"education","lineage":["https://openalex.org/I21250087"]}],"countries":["CZ","IE","NL"],"is_corresponding":false,"raw_author_name":"Milo\u0161 Stanojevi\u0107","raw_affiliation_strings":["Dublin City Univ. ADAPT","Univ. of Amsterdam ILLC","MFF \u00daFAL","Charles Univ. in Prague"],"affiliations":[{"raw_affiliation_string":"Dublin City Univ. ADAPT","institution_ids":["https://openalex.org/I42934936"]},{"raw_affiliation_string":"Univ. of Amsterdam ILLC","institution_ids":["https://openalex.org/I887064364"]},{"raw_affiliation_string":"MFF \u00daFAL","institution_ids":[]},{"raw_affiliation_string":"Charles Univ. in Prague","institution_ids":["https://openalex.org/I21250087"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5077746237"],"corresponding_institution_ids":["https://openalex.org/I21250087","https://openalex.org/I42934936","https://openalex.org/I887064364"],"apc_list":null,"apc_paid":null,"fwci":25.1779,"has_fulltext":true,"cited_by_count":110,"citation_normalized_percentile":{"value":0.99500922,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"199","last_page":"231"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/nist","display_name":"NIST","score":0.8597525358200073},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7813253998756409},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7500761151313782},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.746299147605896},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7061605453491211},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.7015787363052368},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.683324933052063},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5396421551704407},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4948956370353699},{"id":"https://openalex.org/keywords/agreement","display_name":"Agreement","score":0.43679118156433105},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14870432019233704},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.09719029068946838}],"concepts":[{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.8597525358200073},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7813253998756409},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7500761151313782},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.746299147605896},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7061605453491211},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.7015787363052368},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.683324933052063},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5396421551704407},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4948956370353699},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.43679118156433105},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14870432019233704},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.09719029068946838},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":7,"locations":[{"id":"doi:10.18653/v1/w16-2302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2302","pdf_url":"https://www.aclweb.org/anthology/W16-2302.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First Conference on Machine Translation: Volume 2,\n          Shared Task Papers","raw_type":"proceedings-article"},{"id":"pmh:oai:doras.dcu.ie:23367","is_oa":false,"landing_page_url":"http://doras.dcu.ie/23367/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401512","display_name":"Dublin City University Open Access Institutional Repository (Dublin City University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I42934936","host_organization_name":"Dublin City University","host_organization_lineage":["https://openalex.org/I42934936"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":"  Bojar, Ond\u0159ej ORCID: 0000-0002-0606-0050 <https://orcid.org/0000-0002-0606-0050>, Graham, Yvette and Kamran, Amir  (2017) Results of the WMT17 metrics shared task.  In: Second Conference on Machine Translation (WMT17), 7-8 Sept 2017, Copenhagen, Denmark.      ","raw_type":"Conference or Workshop Item"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.396.9161","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.396.9161","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://wing.comp.nus.edu.sg/~antho/W/W13/W13-2202.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.646.4588","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.646.4588","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.aclweb.org/anthology/W/W13/W13-2202v2.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.674.4641","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.674.4641","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.aclweb.org/anthology/W/W14/W14-3336.pdf","raw_type":"text"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:-1160341790992865473","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=-1160341790992865473","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:biblio.ufal.mff.cuni.cz:7810416117753177929","is_oa":true,"landing_page_url":"http://hdl.handle.net/11346/BIBLIO@id=7810416117753177929","pdf_url":null,"source":{"id":"https://openalex.org/S4306401077","display_name":"Institute of Formal and Applied Linguistics (\u00daFAL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"}],"best_oa_location":{"id":"doi:10.18653/v1/w16-2302","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2302","pdf_url":"https://www.aclweb.org/anthology/W16-2302.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the First Conference on Machine Translation: Volume 2,\n          Shared Task Papers","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6299999952316284,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G4956428346","display_name":null,"funder_award_id":"Horizon 2020 research and innovatio","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5036817778","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innov","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5399556803","display_name":null,"funder_award_id":"This project has received funding from the Europea","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G7842005466","display_name":null,"funder_award_id":"Horizon 2020","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8633428685","display_name":null,"funder_award_id":"European Union's Horizon 2020 research and innovat","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8884546066","display_name":"QT21: Quality Translation 21","funder_award_id":"645452","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G8900516709","display_name":"Health in my Language","funder_award_id":"644402","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2508316494.pdf","grobid_xml":"https://content.openalex.org/works/W2508316494.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W20811212","https://openalex.org/W2021095477","https://openalex.org/W2078861931","https://openalex.org/W2101105183","https://openalex.org/W2149327368","https://openalex.org/W2152311128","https://openalex.org/W2167007060","https://openalex.org/W2250234233","https://openalex.org/W2250537251","https://openalex.org/W2250597803","https://openalex.org/W2251117546","https://openalex.org/W2251386554","https://openalex.org/W2251610689","https://openalex.org/W2251926178","https://openalex.org/W2251994258","https://openalex.org/W2252028648","https://openalex.org/W2252035278","https://openalex.org/W2252166243","https://openalex.org/W2257408573","https://openalex.org/W2260677151","https://openalex.org/W2270190199","https://openalex.org/W2294699749","https://openalex.org/W2403821559","https://openalex.org/W2470681840","https://openalex.org/W2471690557","https://openalex.org/W2509854663","https://openalex.org/W2511254834","https://openalex.org/W2511950321","https://openalex.org/W2512848817","https://openalex.org/W2512924740","https://openalex.org/W2513167989","https://openalex.org/W2517773427","https://openalex.org/W2613253298","https://openalex.org/W2917452219"],"related_works":["https://openalex.org/W2106922437","https://openalex.org/W2807901368","https://openalex.org/W2158491338","https://openalex.org/W2133733652","https://openalex.org/W2072658171","https://openalex.org/W2606392311","https://openalex.org/W4385956668","https://openalex.org/W51364034","https://openalex.org/W2898073868","https://openalex.org/W4284663758"],"abstract_inverted_index":{"This":[0],"paper":[1],"presents":[2],"the":[3,6,19,22,27],"results":[4],"of":[5,14,21,35,49,70,84,89,104],"WMT16":[7,28,80],"Metrics":[8],"Shared":[9,29],"Task.":[10,31],"We":[11,32],"asked":[12],"participants":[13],"this":[15],"task":[16],"to":[17,44],"score":[18],"outputs":[20],"MT":[23],"systems":[24],"involved":[25],"in":[26,68,87,100],"Translation":[30],"collected":[33,64],"scores":[34,48,65,77],"16":[36],"metrics":[37,52],"from":[38],"9":[39,50],"research":[40],"groups.":[41],"In":[42],"addition":[43],"that,":[45],"we":[46],"computed":[47],"standard":[51],"(BLEU,":[53],"SentBLEU,":[54],"NIST,":[55],"WER,":[56],"PER,":[57],"TER":[58],"and":[59,86],"CDER)":[60],"as":[61],"baselines.":[62],"The":[63],"were":[66],"evaluated":[67],"terms":[69,88],"system-level":[71],"correlation":[72,92],"(how":[73,93],"well":[74],"each":[75],"metric's":[76],"correlate":[78],"with":[79,98],"official":[81],"manual":[82],"ranking":[83],"systems)":[85],"segment":[90],"level":[91],"often":[94],"a":[95,105],"metric":[96],"agrees":[97],"humans":[99],"comparing":[101],"two":[102],"translations":[103],"particular":[106],"sentence).":[107]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":17},{"year":2018,"cited_by_count":21},{"year":2017,"cited_by_count":13},{"year":2016,"cited_by_count":6},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":3}],"updated_date":"2026-04-21T08:09:41.155169","created_date":"2025-10-10T00:00:00"}