{"id":"https://openalex.org/W7155106236","doi":"https://doi.org/10.48550/arxiv.2604.17393","title":"Who Watches the Watchmen? Humans Disagree With Translation Metrics on Unseen Domains","display_name":"Who Watches the Watchmen? Humans Disagree With Translation Metrics on Unseen Domains","publication_year":2026,"publication_date":"2026-04-19","ids":{"openalex":"https://openalex.org/W7155106236","doi":"https://doi.org/10.48550/arxiv.2604.17393"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.17393","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17393","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.17393","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134144198","display_name":"Finn Schmidt","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schmidt, Finn","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134161359","display_name":"Jan Philip Wahle","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wahle, Jan Philip","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062790326","display_name":"Terry Ruas","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ruas, Terry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134176831","display_name":"Bela Gipp","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gipp, Bela","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6985999941825867,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.6985999941825867,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.10830000042915344,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.02850000001490116,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7757999897003174},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.6626999974250793},{"id":"https://openalex.org/keywords/annotation","display_name":"Annotation","score":0.6040999889373779},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.41659998893737793},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.40529999136924744},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.4034000039100647}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7757999897003174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.75},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.6626999974250793},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6478999853134155},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.6040999889373779},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5461000204086304},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42329999804496765},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.41659998893737793},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.40529999136924744},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.4034000039100647},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.38850000500679016},{"id":"https://openalex.org/C111219384","wikidata":"https://www.wikidata.org/wiki/Q6954384","display_name":"NIST","level":2,"score":0.35040000081062317},{"id":"https://openalex.org/C2776434776","wikidata":"https://www.wikidata.org/wiki/Q19246213","display_name":"Domain adaptation","level":3,"score":0.3264000117778778},{"id":"https://openalex.org/C155092808","wikidata":"https://www.wikidata.org/wiki/Q182557","display_name":"Computational linguistics","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.272599995136261}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.17393","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17393","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.17393","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.17393","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Automatic":[0],"evaluation":[1,51],"metrics":[2,21,118],"are":[3,22],"central":[4],"to":[5,37,122,129,151,161],"the":[6,25,95,125,156],"development":[7],"of":[8,94,165],"machine":[9],"translation":[10,47,98],"systems,":[11,48],"yet":[12],"their":[13,35],"robustness":[14,36,134],"under":[15],"domain":[16,54,104,159],"shift":[17],"remains":[18],"unclear.":[19],"Most":[20],"developed":[23],"on":[24,27,155],"Workshop":[26],"Machine":[28],"Translation":[29],"(WMT)":[30],"benchmarks,":[31],"raising":[32],"concerns":[33],"about":[34],"unseen":[38,44,107,157],"domains.":[39,109,188],"Prior":[40],"studies":[41],"that":[42,116],"analyze":[43],"domains":[45],"vary":[46],"annotators,":[49],"or":[50],"conditions,":[52],"confounding":[53],"effects":[55],"with":[56],"human":[57,75,141],"annotation":[58],"noise.":[59],"To":[60],"address":[61],"these":[62],"biases,":[63],"we":[64,84,113,138],"introduce":[65],"a":[66],"systematic":[67],"multi-annotator":[68],"Cross-Domain":[69],"Error-Span-Annotation":[70],"dataset":[71],"(CD-ESA),":[72],"comprising":[73],"18.8k":[74],"error":[76],"span":[77],"annotations":[78,145],"across":[79,100,186],"three":[80],"language":[81,89],"pairs,":[82],"where":[83],"fix":[85],"annotators":[86],"within":[87],"each":[88],"pair":[90],"and":[91,105],"evaluate":[92],"translations":[93],"same":[96],"six":[97],"systems":[99],"one":[101],"seen":[102],"news":[103],"two":[106],"technical":[108],"Using":[110],"this":[111,133],"dataset,":[112],"first":[114],"find":[115],"automatic":[117],"appear":[119],"surprisingly":[120],"robust":[121],"domain-shifts":[123],"at":[124],"segment":[126],"level":[127],"(up":[128],"0.69":[130],"agreement),":[131],"but":[132],"largely":[135],"disappears":[136],"once":[137],"account":[139],"for":[140],"label":[142],"variation.":[143],"Averaging":[144],"increases":[146],"inter-annotator":[147,175],"agreement":[148,164,173,182],"by":[149],"up":[150],"+0.11.":[152],"Metrics":[153],"struggle":[154],"chemical":[158],"compared":[160],"humans":[162],"(inter-annotator":[163],"0.78-0.83":[166],"vs.":[167],"0.96).":[168],"We":[169],"recommend":[170],"comparing":[171,179],"metric-human":[172,181],"against":[174],"agreement,":[176],"rather":[177],"than":[178],"raw":[180],"alone,":[183],"when":[184],"evaluating":[185],"different":[187]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-04-22T00:00:00"}