{"id":"https://openalex.org/W7166877303","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1145","title":"Who Watches the Watchmen? Humans Disagree With Translation Metrics on Unseen Domains","display_name":"Who Watches the Watchmen? Humans Disagree With Translation Metrics on Unseen Domains","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166877303","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1145"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1145","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1145","pdf_url":"https://aclanthology.org/2026.findings-acl.1145.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1145.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139749638","display_name":"Finn Schmidt","orcid":null},"institutions":[{"id":"https://openalex.org/I74656192","display_name":"University of G\u00f6ttingen","ror":"https://ror.org/01y9bpm73","country_code":"DE","type":"education","lineage":["https://openalex.org/I74656192"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Finn Schmidt","raw_affiliation_strings":["University of G\u00f6ttingen , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of G\u00f6ttingen , Germany","institution_ids":["https://openalex.org/I74656192"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008679414","display_name":"Jan Philip Wahle","orcid":"https://orcid.org/0000-0002-2116-9767"},"institutions":[{"id":"https://openalex.org/I74656192","display_name":"University of G\u00f6ttingen","ror":"https://ror.org/01y9bpm73","country_code":"DE","type":"education","lineage":["https://openalex.org/I74656192"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Jan Philip Wahle","raw_affiliation_strings":["University of G\u00f6ttingen , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of G\u00f6ttingen , Germany","institution_ids":["https://openalex.org/I74656192"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062790326","display_name":"Terry Ruas","orcid":null},"institutions":[{"id":"https://openalex.org/I74656192","display_name":"University of G\u00f6ttingen","ror":"https://ror.org/01y9bpm73","country_code":"DE","type":"education","lineage":["https://openalex.org/I74656192"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Terry Ruas","raw_affiliation_strings":["University of G\u00f6ttingen , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of G\u00f6ttingen , Germany","institution_ids":["https://openalex.org/I74656192"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058837356","display_name":"B\u00e9la Gipp","orcid":"https://orcid.org/0000-0001-6522-3019"},"institutions":[{"id":"https://openalex.org/I74656192","display_name":"University of G\u00f6ttingen","ror":"https://ror.org/01y9bpm73","country_code":"DE","type":"education","lineage":["https://openalex.org/I74656192"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Bela Gipp","raw_affiliation_strings":["University of G\u00f6ttingen , Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of G\u00f6ttingen , Germany","institution_ids":["https://openalex.org/I74656192"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74656192"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.9184953,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"22822","last_page":"22841"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.2409999966621399,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12090","display_name":"Language and cultural evolution","score":0.2409999966621399,"subfield":{"id":"https://openalex.org/subfields/3316","display_name":"Cultural Studies"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.05249999836087227,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11094","display_name":"Face Recognition and Perception","score":0.0494999997317791,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.4422000050544739},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.3073999881744385},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2775999903678894},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.26570001244544983},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.24799999594688416}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5701000094413757},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4652999937534332},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.4422000050544739},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4415999948978424},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3287999927997589},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3073999881744385},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2775999903678894},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.24799999594688416},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.23759999871253967}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1145","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1145","pdf_url":"https://aclanthology.org/2026.findings-acl.1145.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1145","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1145","pdf_url":"https://aclanthology.org/2026.findings-acl.1145.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.4526349902153015,"id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320313139","display_name":"Nieders\u00e4chsische Ministerium f\u00fcr Wissenschaft und Kultur","ror":"https://ror.org/0116z8r77"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166877303.pdf","grobid_xml":"https://content.openalex.org/works/W7166877303.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Automatic":[0],"evaluation":[1,49],"metrics":[2,20,114],"are":[3,21],"central":[4],"to":[5,36,118,125,146,155],"the":[6,24,92,121,150],"development":[7],"of":[8,91,159],"machine":[9],"translation":[10,45,95],"systems,":[11,46],"yet":[12],"their":[13,34],"robustness":[14,35,130],"under":[15],"domain":[16,52,101,153],"shift":[17],"remains":[18],"unclear.Most":[19],"developed":[22],"on":[23,26,149],"Workshop":[25],"Machine":[27],"Translation":[28],"(WMT)":[29],"benchmarks,":[30],"raising":[31],"concerns":[32],"about":[33],"unseen":[37,42,104,151],"domains.Prior":[38],"studies":[39],"that":[40,112],"analyze":[41],"domains":[43],"vary":[44],"annotators,":[47],"or":[48],"conditions,":[50],"confounding":[51],"effects":[53],"with":[54],"human":[55,72,137],"annotation":[56],"noise.To":[57],"address":[58],"these":[59],"biases,":[60],"we":[61,81,109,134],"introduce":[62],"a":[63],"systematic":[64],"multi-annotator":[65],"Cross-Domain":[66],"Error-Span-Annotation":[67],"dataset":[68],"(CD-ESA),":[69],"comprising":[70],"18.8k":[71],"error":[73],"span":[74],"annotations":[75,140],"across":[76,97,178],"three":[77],"language":[78,86],"pairs,":[79],"where":[80],"fix":[82],"annotators":[83],"within":[84],"each":[85],"pair":[87],"and":[88,102],"evaluate":[89],"translations":[90],"same":[93],"six":[94],"systems":[96],"one":[98],"seen":[99],"news":[100],"two":[103],"technical":[105],"domains.Using":[106],"this":[107,129],"dataset,":[108],"first":[110],"find":[111],"automatic":[113],"appear":[115],"surprisingly":[116],"robust":[117],"domain-shifts":[119],"at":[120],"segment":[122],"level":[123],"(up":[124],"0.69":[126],"agreement),":[127],"but":[128],"largely":[131],"disappears":[132],"once":[133],"account":[135],"for":[136],"label":[138],"variation.Averaging":[139],"increases":[141],"inter-annotator":[142,167],"agreement":[143,158,165,174],"by":[144],"up":[145],"+0.11.Metrics":[147],"struggle":[148],"chemical":[152],"compared":[154],"humans":[156],"(interannotator":[157],"0.78-0.83vs.":[160],"0.96).We":[161],"recommend":[162],"comparing":[163,171],"metric-human":[164,173],"against":[166],"agreement,":[168],"rather":[169],"than":[170],"raw":[172],"alone,":[175],"when":[176],"evaluating":[177],"different":[179],"domains.":[180]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}