{"id":"https://openalex.org/W7140152012","doi":null,"title":"Towards better measurement of progress in machine translation: evaluation and meta-evaluation","display_name":"Towards better measurement of progress in machine translation: evaluation and meta-evaluation","publication_year":2026,"publication_date":"2026-01-28","ids":{"openalex":"https://openalex.org/W7140152012"},"language":"en","primary_location":{"id":"pmh:oai:iris.uniroma1.it:11573/1762185","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11573/1762185","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/doctoralThesis"},"volume":null,"type":"dissertation","indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://hdl.handle.net/11573/1762185","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107454193","display_name":"Lorenzo Proietti","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"PROIETTI, LORENZO","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5107454193"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9901999831199646,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0027000000700354576,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.002400000113993883,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-translation-evaluation","display_name":"machine translation evaluation","score":0.984000027179718},{"id":"https://openalex.org/keywords/meta-evaluation","display_name":"meta-evaluation","score":0.9750000238418579},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.7919999957084656},{"id":"https://openalex.org/keywords/performance-saturation","display_name":"performance saturation","score":0.718999981880188},{"id":"https://openalex.org/keywords/benchmark-design","display_name":"benchmark design","score":0.7509999871253967},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"pairwise comparison","score":0.7490000128746033},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8360000252723694},{"id":"https://openalex.org/keywords/progress-measurement","display_name":"progress measurement","score":0.5550000071525574}],"concepts":[{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7669000029563904},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6923999786376953},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.5861999988555908},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5444999933242798},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5077000260353088},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.4893999993801117},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.447299987077713},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.391400009393692},{"id":"https://openalex.org/C2780898871","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Performance metric","level":2,"score":0.38280001282691956},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3199999928474426},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.31929999589920044},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.2937000095844269},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.25679999589920044},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.25450000166893005}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:iris.uniroma1.it:11573/1762185","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11573/1762185","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/doctoralThesis"}],"best_oa_location":{"id":"pmh:oai:iris.uniroma1.it:11573/1762185","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/11573/1762185","pdf_url":null,"source":{"id":"https://openalex.org/S4377196107","display_name":"IRIS Research product catalog (Sapienza University of Rome)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/doctoralThesis"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"id":"https://metadata.un.org/sdg/4","score":0.8116371035575867,"display_name":"Quality education"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reliable":[0],"evaluation":[1,330,380],"is":[2,22,34,40,82],"central":[3],"to":[4,18,35,42,77,85,250,295,315,368],"Machine":[5,28],"Learning.":[6],"Without":[7],"it,":[8],"progress":[9,74,204,300,352],"cannot":[10],"be":[11],"measured,":[12],"claims":[13,273],"of":[14,124,179,203,274],"improvement":[15],"are":[16,151,277],"hard":[17],"verify,":[19],"and":[20,46,73,97,109,114,128,145,148,169,172,192,197,207,235,257,260,270,298,302,323,356,378],"research":[21,171],"misdirected.":[23],"This":[24],"thesis":[25,183],"focuses":[26],"on":[27,62,262,341,383],"Translation":[29],"(MT),":[30],"where":[31],"the":[32,43,49,66,122,161,195,232,263],"goal":[33],"produce":[36],"a":[37,87,238,305,332],"translation":[38,119],"that":[39,200,221,241,272,282,312,320,358,372],"faithful":[41],"source":[44,89,310],"meaning":[45],"fluent":[47],"in":[48,121,133,174,186,214,353],"target":[50],"language.":[51],"As":[52],"modern":[53,284],"MT":[54,81,154,217,286,354],"systems":[55,70,376],"reach":[56],"very":[57],"high":[58],"quality,":[59],"often":[60],"relying":[61],"Large":[63],"Language":[64],"Models,":[65],"differences":[67],"between":[68,335],"top":[69],"become":[71,291],"subtle":[72],"increasingly":[75],"difficult":[76,318],"measure.":[78],"Compounding":[79],"this,":[80],"inherently":[83],"challenging":[84,374],"evaluate:":[86],"single":[88],"text":[90],"may":[91],"admit":[92],"many":[93,101,125],"equally":[94],"acceptable":[95,126],"translations,":[96,338],"errors":[98],"can":[99,165,223],"take":[100],"forms.":[102],"Collectively,":[103],"these":[104,180,243,349],"conditions":[105],"affect":[106],"both":[107],"human":[108,143,251,255,275,369],"automatic":[110,115,175,329,379],"evaluation.":[111,176],"Human":[112],"annotators":[113],"metrics":[116,159,230,259],"must":[117,135],"assess":[118],"quality":[120],"presence":[123],"outputs":[127,343],"diverse":[129],"error":[130],"patterns.":[131],"Meta-evaluation,":[132],"turn,":[134],"determine":[136],"how":[137,146],"well":[138],"metric":[139,167,225,248,365],"scores":[140],"align":[141],"with":[142,381],"judgments":[144],"informative":[147],"accurate":[149],"they":[150,222],"for":[152,160,189,193,231,283,307],"comparing":[153],"systems;":[155],"if":[156],"it":[157,164],"rewards":[158],"wrong":[162,233],"reasons,":[163,234],"distort":[166,224],"rankings":[168,226,249,366],"misdirect":[170],"practices":[173],"In":[177],"light":[178],"challenges,":[181],"this":[182],"examines":[184],"weaknesses":[185],"current":[187],"methodologies":[188],"evaluating":[190,194],"translations":[191],"evaluators,":[196],"proposes":[198],"solutions":[199],"keep":[201,351],"measurements":[202],"fair,":[205],"robust,":[206],"informative.":[208],"First,":[209],"we":[210,236,246,280,303,313,327],"identify":[211],"fundamental":[212],"flaws":[213],"widely":[215],"used":[216],"meta-evaluation":[218,360],"strategies,":[219],"showing":[220,271],"by":[227,253],"inadvertently":[228],"rewarding":[229],"introduce":[237,304],"revised":[239],"protocol":[240],"corrects":[242],"effects.":[244],"Second,":[245],"anchor":[247],"performance":[252,267,296],"estimating":[254],"baselines":[256],"placing":[258],"humans":[261],"same":[264],"scale,":[265],"making":[266],"headroom":[268],"explicit,":[269],"parity":[276],"fragile.":[278],"Third,":[279],"show":[281],"high-performing":[285],"systems,":[287],"standard":[288],"benchmarks":[289,319,371],"have":[290],"too":[292],"easy,":[293],"leading":[294],"saturation":[297,322],"blunting":[299],"measurement,":[301],"methodology":[306],"identifying":[308],"hard-to-translate":[309],"texts":[311],"use":[314],"construct":[316],"more":[317],"resist":[321],"preserve":[324],"headroom.":[325],"Finally,":[326],"reframe":[328],"as":[331,375],"pairwise":[333],"comparison":[334],"two":[336],"candidate":[337],"improving":[339],"accuracy":[340],"high-quality":[342,384],"while":[344],"remaining":[345],"efficient.":[346],"Taken":[347],"together,":[348],"contributions":[350],"measurable":[355],"trustworthy,":[357],"is,":[359],"reflecting":[361],"genuine":[362],"evaluative":[363],"ability,":[364],"anchored":[367],"performance,":[370],"remain":[373],"improve,":[377],"resolution":[382],"outputs.":[385]},"counts_by_year":[],"updated_date":"2026-09-25T08:18:29.962427","created_date":"2026-03-24T00:00:00"}