{"id":"https://openalex.org/W4416033518","doi":"https://doi.org/10.18653/v1/2025.wmt-1.19","title":"GAMBIT+: A Challenge Set for Evaluating Gender Bias in Machine Translation Quality Estimation Metrics","display_name":"GAMBIT+: A Challenge Set for Evaluating Gender Bias in Machine Translation Quality Estimation Metrics","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416033518","doi":"https://doi.org/10.18653/v1/2025.wmt-1.19"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2025.wmt-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.wmt-1.19","pdf_url":"https://aclanthology.org/2025.wmt-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Conference on Machine Translation","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.wmt-1.19.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067913427","display_name":"George Filandrianos","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"George Filandrianos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032386720","display_name":"Orfeas Menis Mastromichalakis","orcid":"https://orcid.org/0000-0001-5286-1388"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Orfeas Menis Mastromichalakis","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014587525","display_name":"Wafaa Mohammed","orcid":"https://orcid.org/0000-0002-4067-8217"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wafaa Mohammed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109379051","display_name":"Giuseppe Attanasio","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Giuseppe Attanasio","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5004372721","display_name":"Chrysoula Zerva","orcid":"https://orcid.org/0000-0002-4031-9492"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chrysoula Zerva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"314","last_page":"326"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5514000058174133,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.5514000058174133,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12380","display_name":"Authorship Attribution and Profiling","score":0.09120000153779984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.09019999951124191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5295000076293945},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.46219998598098755},{"id":"https://openalex.org/keywords/estimation","display_name":"Estimation","score":0.43869999051094055},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.3587000072002411},{"id":"https://openalex.org/keywords/gender-bias","display_name":"Gender bias","score":0.3481000065803528},{"id":"https://openalex.org/keywords/data-set","display_name":"Data set","score":0.32600000500679016}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6330000162124634},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5870000123977661},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5295000076293945},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.46219998598098755},{"id":"https://openalex.org/C96250715","wikidata":"https://www.wikidata.org/wiki/Q965330","display_name":"Estimation","level":2,"score":0.43869999051094055},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4350000023841858},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.3587000072002411},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35530000925064087},{"id":"https://openalex.org/C2983427547","wikidata":"https://www.wikidata.org/wiki/Q93200","display_name":"Gender bias","level":2,"score":0.3481000065803528},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3402000069618225},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.32600000500679016},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.302700012922287},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.25920000672340393},{"id":"https://openalex.org/C3020001037","wikidata":"https://www.wikidata.org/wiki/Q836575","display_name":"Quality assessment","level":3,"score":0.2581000030040741},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.25600001215934753},{"id":"https://openalex.org/C2781170535","wikidata":"https://www.wikidata.org/wiki/Q30587856","display_name":"Noisy data","level":2,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.wmt-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.wmt-1.19","pdf_url":"https://aclanthology.org/2025.wmt-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Conference on Machine Translation","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.wmt-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.wmt-1.19","pdf_url":"https://aclanthology.org/2025.wmt-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Conference on Machine Translation","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2074083719","display_name":null,"funder_award_id":"50008","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G2124936668","display_name":"European Lighthouse of AI for Sustainability","funder_award_id":"101120237","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G273553189","display_name":null,"funder_award_id":"UID/50008","funder_id":"https://openalex.org/F4320328988","funder_display_name":"Instituto de Telecomunica\u00e7\u00f5es"},{"id":"https://openalex.org/G3232057226","display_name":null,"funder_award_id":"101120237","funder_id":"https://openalex.org/F4320334322","funder_display_name":"HORIZON EUROPE Framework Programme"},{"id":"https://openalex.org/G3850789590","display_name":null,"funder_award_id":"50008","funder_id":"https://openalex.org/F4320334779","funder_display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia"},{"id":"https://openalex.org/G3884855522","display_name":null,"funder_award_id":"UID/50008","funder_id":"https://openalex.org/F4320334779","funder_display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia"},{"id":"https://openalex.org/G5298389108","display_name":"Unified Transcription and Translation for Extended Reality","funder_award_id":"101070631","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G5475635744","display_name":null,"funder_award_id":"UID/50008","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G670063171","display_name":null,"funder_award_id":"C645008882-00000055","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320328988","display_name":"Instituto de Telecomunica\u00e7\u00f5es","ror":"https://ror.org/02ht4fk33"},{"id":"https://openalex.org/F4320334322","display_name":"HORIZON EUROPE Framework Programme","ror":null},{"id":"https://openalex.org/F4320334779","display_name":"Funda\u00e7\u00e3o para a Ci\u00eancia e a Tecnologia","ror":"https://ror.org/00snfqn58"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416033518.pdf","grobid_xml":"https://content.openalex.org/works/W4416033518.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Gender":[0],"bias":[1,12,167],"in":[2,13,102,117],"machine":[3],"translation":[4],"(MT)":[5],"systems":[6],"has":[7],"been":[8],"extensively":[9],"documented,":[10],"but":[11],"automatic":[14],"quality":[15],"estimation":[16],"(QE)":[17],"metrics":[18,26,64],"remains":[19],"comparatively":[20],"underexplored.Existing":[21],"studies":[22],"suggest":[23],"that":[24,89],"QE":[25,63,136],"can":[27],"also":[28],"exhibit":[29],"gender":[30,120],"bias,":[31],"yet":[32],"most":[33],"analyses":[34],"are":[35,90],"limited":[36],"by":[37,169],"small":[38],"datasets,":[39],"narrow":[40],"occupational":[41,70,123],"coverage,":[42],"and":[43,94,150,171],"restricted":[44],"language":[45,105],"variety.To":[46],"address":[47],"this":[48],"gap,":[49],"we":[50,82],"introduce":[51],"a":[52],"large-scale":[53],"challenge":[54],"set":[55,157],"specifically":[56],"designed":[57],"to":[58,85,144],"probe":[59],"the":[60,73,118,122,155],"behavior":[61],"of":[62,76,121,158],"when":[65],"evaluating":[66],"translations":[67],"containing":[68],"gender-ambiguous":[69,80],"terms.Building":[71],"on":[72],"GAMBIT":[74],"corpus":[75],"English":[77],"texts":[78,159],"with":[79,98,111,128],"occupations,":[81],"extend":[83],"coverage":[84],"three":[86],"source":[87,107],"languages":[88,97],"genderless":[91],"or":[92,141],"natural-gendered,":[93],"eleven":[95],"target":[96,113],"grammatical":[99,119,131],"gender,":[100],"resulting":[101],"33":[103],"source-target":[104],"pairs.Each":[106],"text":[108],"is":[109,160],"paired":[110],"two":[112],"versions":[114],"differing":[115],"only":[116],"term(s)":[124],"(masculine":[125],"vs.":[126],"feminine),":[127],"all":[129,163],"dependent":[130],"elements":[132],"adjusted":[133],"accordingly.An":[134],"unbiased":[135],"metric":[137],"should":[138],"assign":[139],"equal":[140],"near-equal":[142],"scores":[143],"both":[145],"versions.The":[146],"dataset's":[147],"scale,":[148],"breadth,":[149],"fully":[151],"parallel":[152],"design,":[153],"where":[154],"same":[156],"aligned":[161],"across":[162,174],"languages,":[164],"enables":[165],"fine-grained":[166],"analysis":[168],"occupation":[170],"systematic":[172],"comparisons":[173],"languages.":[175]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-08T00:00:00"}