{"id":"https://openalex.org/W2250830017","doi":"https://doi.org/10.18653/v1/w15-3059","title":"How do Humans Evaluate Machine Translation","display_name":"How do Humans Evaluate Machine Translation","publication_year":2015,"publication_date":"2015-01-01","ids":{"openalex":"https://openalex.org/W2250830017","doi":"https://doi.org/10.18653/v1/w15-3059","mag":"2250830017"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w15-3059","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-3059","pdf_url":"https://www.aclweb.org/anthology/W15-3059.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W15-3059.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5113551755","display_name":"Francisco Guzm\u00e1n","orcid":null},"institutions":[{"id":"https://openalex.org/I92528248","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05","country_code":"QA","type":"funder","lineage":["https://openalex.org/I92528248"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Francisco Guzm\u00e1n","raw_affiliation_strings":["QATAR FOUNDATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"QATAR FOUNDATION","institution_ids":["https://openalex.org/I92528248"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047182137","display_name":"Ahmed Abdelal\u00ed","orcid":"https://orcid.org/0000-0002-4160-8181"},"institutions":[{"id":"https://openalex.org/I92528248","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05","country_code":"QA","type":"funder","lineage":["https://openalex.org/I92528248"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Ahmed Abdelali","raw_affiliation_strings":["ALT Research Group Qatar Computing Research Institute, HBKU","QATAR FOUNDATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALT Research Group Qatar Computing Research Institute, HBKU","institution_ids":[]},{"raw_affiliation_string":"QATAR FOUNDATION","institution_ids":["https://openalex.org/I92528248"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001019197","display_name":"Irina Temnikova","orcid":null},"institutions":[{"id":"https://openalex.org/I92528248","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05","country_code":"QA","type":"funder","lineage":["https://openalex.org/I92528248"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Irina Temnikova","raw_affiliation_strings":["ALT Research Group Qatar Computing Research Institute, HBKU","QATAR FOUNDATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALT Research Group Qatar Computing Research Institute, HBKU","institution_ids":[]},{"raw_affiliation_string":"QATAR FOUNDATION","institution_ids":["https://openalex.org/I92528248"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042954793","display_name":"Hassan Sajjad","orcid":"https://orcid.org/0000-0002-8584-6595"},"institutions":[{"id":"https://openalex.org/I92528248","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05","country_code":"QA","type":"funder","lineage":["https://openalex.org/I92528248"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Hassan Sajjad","raw_affiliation_strings":["ALT Research Group Qatar Computing Research Institute, HBKU","QATAR FOUNDATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALT Research Group Qatar Computing Research Institute, HBKU","institution_ids":[]},{"raw_affiliation_string":"QATAR FOUNDATION","institution_ids":["https://openalex.org/I92528248"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064572063","display_name":"Stephan Vogel","orcid":"https://orcid.org/0000-0003-2630-4881"},"institutions":[{"id":"https://openalex.org/I92528248","display_name":"Qatar Foundation","ror":"https://ror.org/01cawbq05","country_code":"QA","type":"funder","lineage":["https://openalex.org/I92528248"]}],"countries":["QA"],"is_corresponding":false,"raw_author_name":"Stephan Vogel","raw_affiliation_strings":["ALT Research Group Qatar Computing Research Institute, HBKU","QATAR FOUNDATION"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ALT Research Group Qatar Computing Research Institute, HBKU","institution_ids":[]},{"raw_affiliation_string":"QATAR FOUNDATION","institution_ids":["https://openalex.org/I92528248"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.671,"has_fulltext":true,"cited_by_count":14,"citation_normalized_percentile":{"value":0.9200948,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"457","last_page":"466"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7385269403457642},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.694563090801239},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5174082517623901},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5086420774459839},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5071923732757568},{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.5013151168823242},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.47134777903556824},{"id":"https://openalex.org/keywords/eye-tracking","display_name":"Eye tracking","score":0.4306762218475342},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.4178466796875},{"id":"https://openalex.org/keywords/task-analysis","display_name":"Task analysis","score":0.41428685188293457},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07351166009902954}],"concepts":[{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7385269403457642},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.694563090801239},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5174082517623901},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5086420774459839},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5071923732757568},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.5013151168823242},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.47134777903556824},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.4306762218475342},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.4178466796875},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.41428685188293457},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07351166009902954},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w15-3059","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-3059","pdf_url":"https://www.aclweb.org/anthology/W15-3059.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w15-3059","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w15-3059","pdf_url":"https://www.aclweb.org/anthology/W15-3059.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Tenth Workshop on Statistical Machine Translation","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.8600000143051147,"id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2250830017.pdf","grobid_xml":"https://content.openalex.org/works/W2250830017.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W198014083","https://openalex.org/W567326294","https://openalex.org/W1489409710","https://openalex.org/W1516288603","https://openalex.org/W2016336271","https://openalex.org/W2030073608","https://openalex.org/W2058941677","https://openalex.org/W2087735403","https://openalex.org/W2115081467","https://openalex.org/W2139940533","https://openalex.org/W2147192413","https://openalex.org/W2148504207","https://openalex.org/W2149327368","https://openalex.org/W2167007060","https://openalex.org/W2250188786","https://openalex.org/W2251476899","https://openalex.org/W2331726854","https://openalex.org/W2582743722","https://openalex.org/W3037252522","https://openalex.org/W4299687266"],"related_works":["https://openalex.org/W1564680838","https://openalex.org/W2003125260","https://openalex.org/W2060591604","https://openalex.org/W2166791242","https://openalex.org/W3012440055","https://openalex.org/W2585162246","https://openalex.org/W2034202275","https://openalex.org/W1934413089","https://openalex.org/W1992291644","https://openalex.org/W2098419343"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,41],"take":[4,80],"a":[5,14,91],"closer":[6],"look":[7],"at":[8],"the":[9,24,35,85],"MT":[10,105],"evaluation":[11,25],"process":[12],"from":[13],"glass-box":[15],"perspective":[16],"using":[17,44],"eye-tracking.":[18],"We":[19],"analyze":[20],"two":[21],"aspects":[22],"of":[23,29,37,75,87],"task":[26],"-the":[27],"background":[28],"evaluators":[30,77],"(monolingual":[31],"or":[32],"bilingual)":[33],"and":[34,40,46,83,102],"sources":[36,74],"information":[38,67],"available,":[39],"evaluate":[42],"them":[43],"time":[45,82],"consistency":[47],"as":[48],"criteria.":[49],"Our":[50,95],"findings":[51,96],"show":[52],"that":[53,98],"monolinguals":[54,112],"are":[55],"slower":[56],"but":[57],"more":[58,81],"consistent":[59,101],"than":[60],"bilinguals,":[61],"especially":[62],"when":[63],"only":[64,114],"target":[65,115],"language":[66,116],"is":[68,90,108],"available.":[69],"When":[70],"exposed":[71],"to":[72,99,110],"various":[73],"information,":[76],"in":[78,84,93],"general":[79],"case":[86],"monolinguals,":[88],"there":[89],"drop":[92],"consistency.":[94],"suggest":[97],"have":[100],"cost":[103],"effective":[104],"evaluations,":[106],"it":[107],"better":[109],"use":[111],"with":[113],"information.":[117]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":2},{"year":2016,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}