{"id":"https://openalex.org/W4402638667","doi":"https://doi.org/10.1162/coli_a_00537","title":"Machine Translation Meta Evaluation through Translation Accuracy Challenge Sets","display_name":"Machine Translation Meta Evaluation through Translation Accuracy Challenge Sets","publication_year":2024,"publication_date":"2024-09-19","ids":{"openalex":"https://openalex.org/W4402638667","doi":"https://doi.org/10.1162/coli_a_00537"},"language":"en","primary_location":{"id":"doi:10.1162/coli_a_00537","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00537","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/coli_a_00537","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001787620","display_name":"Nikita Moghe","orcid":null},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Nikita Moghe","raw_affiliation_strings":["University of Edinburgh, School of Informatics. nikitamoghe29@gmail.com","School of Informatics, University of Edinburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, School of Informatics. nikitamoghe29@gmail.com","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"School of Informatics, University of Edinburgh","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093826687","display_name":"Arnisa Fazla","orcid":null},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Arnisa Fazla","raw_affiliation_strings":["University of Zurich, Department of Computational Linguistics. arnisa.fazla@uzh.ch","Department of Computational Linguistics, University of Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zurich, Department of Computational Linguistics. arnisa.fazla@uzh.ch","institution_ids":[]},{"raw_affiliation_string":"Department of Computational Linguistics, University of Zurich","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086632463","display_name":"Chantal Amrhein","orcid":null},"institutions":[{"id":"https://openalex.org/I188703185","display_name":"GTx (United States)","ror":"https://ror.org/0400j3f74","country_code":"US","type":"company","lineage":["https://openalex.org/I188703185"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chantal Amrhein","raw_affiliation_strings":["Supertext. chantal@supertext.ch","Textshuttle, Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Supertext. chantal@supertext.ch","institution_ids":["https://openalex.org/I188703185"]},{"raw_affiliation_string":"Textshuttle, Zurich","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035898384","display_name":"Tom Kocmi","orcid":"https://orcid.org/0000-0002-7993-9859"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB","US"],"is_corresponding":false,"raw_author_name":"Tom Kocmi","raw_affiliation_strings":["Microsoft. tom.kocmi@microsoft.com","Microsoft"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft. tom.kocmi@microsoft.com","institution_ids":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055023881","display_name":"Mark Steedman","orcid":"https://orcid.org/0000-0003-2509-0797"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Mark Steedman","raw_affiliation_strings":["University of Edinburgh, School of Informatics. m.steedman@ed.ac.uk","School of Informatics, University of Edinburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, School of Informatics. m.steedman@ed.ac.uk","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"School of Informatics, University of Edinburgh","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038456766","display_name":"Alexandra Birch","orcid":"https://orcid.org/0000-0002-9022-3405"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Alexandra Birch","raw_affiliation_strings":["University of Edinburgh, School of Informatics. a.birch@ed.ac.uk","School of Informatics, University of Edinburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, School of Informatics. a.birch@ed.ac.uk","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"School of Informatics, University of Edinburgh","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005771535","display_name":"Rico Sennrich","orcid":"https://orcid.org/0000-0002-1438-4741"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Rico Sennrich","raw_affiliation_strings":["University of Zurich, Department of Computational Linguistics. sennrich@cl.uzh.ch","Department of Computational Linguistics, University of Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zurich, Department of Computational Linguistics. sennrich@cl.uzh.ch","institution_ids":[]},{"raw_affiliation_string":"Department of Computational Linguistics, University of Zurich","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023393552","display_name":"Liane Guillou","orcid":"https://orcid.org/0009-0001-3339-6334"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Liane Guillou","raw_affiliation_strings":["University of Edinburgh, School of Informatics. liane.guillou@ed.ac.uk","School of Informatics, University of Edinburgh"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh, School of Informatics. liane.guillou@ed.ac.uk","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"School of Informatics, University of Edinburgh","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.225,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.82339863,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":99},"biblio":{"volume":"51","issue":"1","first_page":"73","last_page":"137"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9581000208854675,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9581000208854675,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9085999727249146,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8158462047576904},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.7459138035774231},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.7036675810813904},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5577474236488342},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5311257839202881}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8158462047576904},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.7459138035774231},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.7036675810813904},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5577474236488342},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5311257839202881},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/coli_a_00537","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00537","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:0ccdb87981f142e6a376fa21c3897388","is_oa":false,"landing_page_url":"https://doaj.org/article/0ccdb87981f142e6a376fa21c3897388","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics (2024)","raw_type":"article"},{"id":"pmh:oai:pure.ed.ac.uk:openaire/9bba3cf1-0fcd-4e31-b69e-0f42bf78f83f","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/en/publications/9bba3cf1-0fcd-4e31-b69e-0f42bf78f83f","pdf_url":null,"source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Moghe, N, Fazla, A, Amrhein, C, Kocmi, T, Steedman, M, Birch, A, Sennrich, R & Guillou, L 2024, 'Machine translation meta evaluation through translation accuracy challenge sets', Computational Linguistics, pp. 1-60. https://doi.org/10.1162/coli_a_00537","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:pure.ed.ac.uk:publications/9bba3cf1-0fcd-4e31-b69e-0f42bf78f83f","is_oa":true,"landing_page_url":"https://www.research.ed.ac.uk/files/480272035/MogheEtalCL2024MachineTranslationMetaEvaluation.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4406922455","display_name":"Edinburgh Research Explorer","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Moghe, N, Fazla, A, Amrhein, C, Kocmi, T, Steedman, M, Birch, A, Sennrich, R & Guillou, L 2024, 'Machine translation meta evaluation through translation accuracy challenge sets', Computational Linguistics, pp. 1-60. https://doi.org/10.1162/coli_a_00537","raw_type":"info:eu-repo/semantics/publishedVersion"}],"best_oa_location":{"id":"doi:10.1162/coli_a_00537","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_a_00537","pdf_url":null,"source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":98,"referenced_works":["https://openalex.org/W1654441844","https://openalex.org/W2060377713","https://openalex.org/W2101105183","https://openalex.org/W2111780752","https://openalex.org/W2152311128","https://openalex.org/W2512924740","https://openalex.org/W2573119710","https://openalex.org/W2624521690","https://openalex.org/W2739810148","https://openalex.org/W2758950307","https://openalex.org/W2789441927","https://openalex.org/W2792120626","https://openalex.org/W2793353517","https://openalex.org/W2804897457","https://openalex.org/W2891555348","https://openalex.org/W2902545332","https://openalex.org/W2902582221","https://openalex.org/W2952328691","https://openalex.org/W2962784628","https://openalex.org/W2963250244","https://openalex.org/W2963526187","https://openalex.org/W2963641561","https://openalex.org/W2963969878","https://openalex.org/W2964130895","https://openalex.org/W2964270503","https://openalex.org/W2970558573","https://openalex.org/W2970752815","https://openalex.org/W2970854433","https://openalex.org/W2970986500","https://openalex.org/W2971141904","https://openalex.org/W3036677371","https://openalex.org/W3105214104","https://openalex.org/W3152698349","https://openalex.org/W3153592532","https://openalex.org/W3159892921","https://openalex.org/W3169369929","https://openalex.org/W3184561823","https://openalex.org/W3212978464","https://openalex.org/W4225116116","https://openalex.org/W4255340738","https://openalex.org/W4285060763","https://openalex.org/W4285233909","https://openalex.org/W4309674289","https://openalex.org/W4324134461","https://openalex.org/W4376645554","https://openalex.org/W4385570091","https://openalex.org/W4385570703","https://openalex.org/W4385571232","https://openalex.org/W4385571265","https://openalex.org/W4385573754","https://openalex.org/W4389518754","https://openalex.org/W4389518809","https://openalex.org/W4389519109","https://openalex.org/W4389519249","https://openalex.org/W4389519433","https://openalex.org/W4389519504","https://openalex.org/W4389519548","https://openalex.org/W4389523995","https://openalex.org/W4389524000","https://openalex.org/W4389524234","https://openalex.org/W4389524403","https://openalex.org/W4389524415","https://openalex.org/W4389524423","https://openalex.org/W4389524427","https://openalex.org/W4389524545","https://openalex.org/W4401042450","https://openalex.org/W4402227281","https://openalex.org/W6600880057","https://openalex.org/W6629155019","https://openalex.org/W6636126286","https://openalex.org/W6732522479","https://openalex.org/W6752572194","https://openalex.org/W6753379416","https://openalex.org/W6755207826","https://openalex.org/W6756755370","https://openalex.org/W6761205521","https://openalex.org/W6784395031","https://openalex.org/W6784447870","https://openalex.org/W6785147540","https://openalex.org/W6787335730","https://openalex.org/W6799087104","https://openalex.org/W6811034136","https://openalex.org/W6839734537","https://openalex.org/W6847076894","https://openalex.org/W6847118041","https://openalex.org/W6848543228","https://openalex.org/W6849846037","https://openalex.org/W6850603086","https://openalex.org/W6852947401","https://openalex.org/W6853308384","https://openalex.org/W6853431315","https://openalex.org/W6854326685","https://openalex.org/W6854866820","https://openalex.org/W6856652384","https://openalex.org/W6857485937","https://openalex.org/W7000626104","https://openalex.org/W7028504196","https://openalex.org/W7067712623"],"related_works":["https://openalex.org/W2775554247","https://openalex.org/W2883671469","https://openalex.org/W2728761353","https://openalex.org/W2110168585","https://openalex.org/W3107474891","https://openalex.org/W2250213760","https://openalex.org/W4386247111","https://openalex.org/W4327642362","https://openalex.org/W2587014613","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Abstract":[0],"Recent":[1],"machine":[2],"translation":[3,96],"(MT)":[4],"metrics":[5,37,92,129,138,221,226,283,313],"calibrate":[6],"their":[7,144],"effectiveness":[8],"by":[9,20,124,172],"correlating":[10],"with":[11,201,217],"human":[12],"judgment.":[13],"However,":[14],"these":[15,36,282],"results":[16,194],"are":[17,44,55,161,208],"often":[18],"obtained":[19],"averaging":[21],"predictions":[22],"across":[23,38],"large":[24,157],"test":[25],"sets":[26,43],"without":[27],"any":[28],"insights":[29],"into":[30],"the":[31,106,132,167,223,242,248,318,329,334],"strengths":[32],"and":[33,60,116,135,183,186,190,204,233,270,332],"weaknesses":[34],"of":[35,50,68,74,149,169,180,214,296,309],"different":[39,197,202],"error":[40,264,278,306],"types.":[41],"Challenge":[42],"used":[45],"to":[46,109,131,146,228,239,247,262,275,314],"probe":[47],"specific":[48],"dimensions":[49],"metric":[51,198],"behavior":[52],"but":[53],"there":[54],"very":[56],"few":[57],"such":[58],"datasets":[59],"they":[61],"either":[62],"focus":[63,316],"on":[64,114,127,235,289,305,317,322,328],"a":[65,71,80,121,147,174,178,212,294],"limited":[66,72],"number":[67,73,213],"phenomena":[69,100,182,203],"or":[70],"language":[75,86,158,184,244],"pairs.":[76],"We":[77,119,141,152,210],"introduce":[78],"ACES,":[79],"contrastive":[81],"challenge":[82],"set":[83,295],"spanning":[84],"146":[85],"pairs,":[87],"aimed":[88],"at":[89,105],"discovering":[90],"whether":[91],"can":[93],"identify":[94],"68":[95],"accuracy":[97],"errors.":[98],"These":[99],"range":[101,148,179],"from":[102],"basic":[103],"alterations":[104],"word/character":[107],"level":[108,231],"more":[110],"intricate":[111],"errors":[112],"based":[113],"discourse":[115],"real-world":[117],"knowledge.":[118],"conducted":[120],"large-scale":[122],"study":[123],"benchmarking":[125],"ACES":[126,261],"47":[128],"submitted":[130],"WMT":[133,136],"2022":[134],"2023":[137],"shared":[139],"tasks.":[140],"also":[142,284],"measure":[143],"sensitivity":[145],"linguistic":[150,181],"phenomena.":[151],"further":[153,252],"investigate":[154],"claims":[155],"that":[156,176,196,205,281],"models":[159],"(LLMs)":[160],"effective":[162],"as":[163,268],"MT":[164,301],"evaluators,":[165],"addressing":[166],"limitations":[168],"previous":[170],"studies":[171],"using":[173],"dataset":[175,274],"covers":[177],"pairs":[185],"includes":[187],"both":[188],"low-":[189],"medium-resource":[191],"languages.":[192],"Our":[193],"demonstrate":[195],"families":[199],"struggle":[200],"LLM-based":[206],"methods":[207],"unreliable.":[209],"expose":[211],"major":[215],"flaws":[216],"existing":[218],"methods:":[219],"Most":[220],"ignore":[222],"source":[224,249,319],"sentence;":[225],"tend":[227],"prefer":[229],"surface":[230],"overlap;":[232],"over-reliance":[234],"language-agnostic":[236],"representations":[237],"leads":[238],"confusion":[240],"when":[241],"target":[243],"is":[245],"similar":[246],"language.":[250],"To":[251],"encourage":[253],"detailed":[254],"evaluation":[255],"beyond":[256],"singular":[257],"scores,":[258,310],"we":[259,271,292],"expand":[260],"include":[263],"span":[265],"annotations,":[266],"denoted":[267],"SPAN-ACES,":[269],"use":[272],"this":[273],"evaluate":[276],"span-based":[277],"metrics,":[279,302],"showing":[280],"need":[285],"considerable":[286],"improvement.":[287],"Based":[288],"our":[290],"observations,":[291],"provide":[293],"recommendations":[297],"for":[298,338],"building":[299],"better":[300],"including":[303],"focusing":[304,321],"labels":[307],"instead":[308],"ensembling,":[311],"designing":[312],"explicitly":[315],"sentence,":[320],"semantic":[323],"content":[324],"rather":[325],"than":[326],"relying":[327],"lexical":[330],"overlap,":[331],"choosing":[333],"right":[335],"pre-trained":[336],"model":[337],"obtaining":[339],"representations.":[340]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":2}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2025-10-10T00:00:00"}