{"id":"https://openalex.org/W1594025044","doi":null,"title":"Methods for human evaluation of machine translation","display_name":"Methods for human evaluation of machine translation","publication_year":2010,"publication_date":"2010-01-01","ids":{"openalex":"https://openalex.org/W1594025044","mag":"1594025044"},"language":"en","primary_location":{"id":"mag:1594025044","doi":null,"is_oa":false,"landing_page_url":"http://stp.lingfil.uu.se/~sara/papers/sltc10.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306420436","display_name":"Language and Technology Conference","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Language and Technology Conference","raw_type":null},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5078866500","display_name":"Sofia Bremin","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sofia Bremin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5000703759","display_name":"Hongzhan Hu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongzhan Hu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031943611","display_name":"Johanna Karlsson","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Johanna Karlsson","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053488707","display_name":"Anna Prytz Lillkull","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anna Prytz Lillkull","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018886479","display_name":"Martin Wester","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Martin Wester","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037659978","display_name":"Henrik Danielsson","orcid":"https://orcid.org/0000-0002-0446-0827","observed_orcids":["https://orcid.org/0000-0002-0446-0827"]},"institutions":[{"id":"https://openalex.org/I187531555","display_name":"Lund University","ror":"https://ror.org/012a77v79","country_code":"SE","type":"education","lineage":["https://openalex.org/I187531555"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Henrik Danielsson","raw_affiliation_strings":["Lund University, Lund, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-0446-0827","affiliations":[{"raw_affiliation_string":"Lund University, Lund, Sweden","institution_ids":["https://openalex.org/I187531555"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051985869","display_name":"Sara Stymne","orcid":"https://orcid.org/0000-0003-3726-9399","observed_orcids":["https://orcid.org/0000-0003-3726-9399"]},"institutions":[{"id":"https://openalex.org/I33976269","display_name":"Xerox (France)","ror":"https://ror.org/033q0mv79","country_code":"FR","type":"company","lineage":["https://openalex.org/I33976269","https://openalex.org/I4210132870"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Sara Stymne","raw_affiliation_strings":["Xerox (France), Saint-Denis, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xerox (France), Saint-Denis, France","institution_ids":["https://openalex.org/I33976269"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2848,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.38010887,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":94},"biblio":{"volume":null,"issue":null,"first_page":"47","last_page":"48"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.005400000140070915,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-translation-evaluation","display_name":"machine translation evaluation","score":0.9819999933242798},{"id":"https://openalex.org/keywords/human-evaluation","display_name":"human evaluation","score":0.9390000104904175},{"id":"https://openalex.org/keywords/reading-comprehension","display_name":"reading comprehension","score":0.8420000076293945},{"id":"https://openalex.org/keywords/eye-tracking","display_name":"eye tracking","score":0.8999999761581421},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.7080000042915344},{"id":"https://openalex.org/keywords/error-analysis","display_name":"error analysis","score":0.7670000195503235},{"id":"https://openalex.org/keywords/statistical-machine-translation","display_name":"statistical machine translation","score":0.8299999833106995},{"id":"https://openalex.org/keywords/translation-quality","display_name":"translation quality","score":0.4749999940395355},{"id":"https://openalex.org/keywords/system-ranking","display_name":"system ranking","score":0.3959999978542328}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7758578062057495},{"id":"https://openalex.org/C2777413886","wikidata":"https://www.wikidata.org/wiki/Q3276013","display_name":"Fluency","level":2,"score":0.6563117504119873},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.6218726634979248},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6213618516921997},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6121330261230469},{"id":"https://openalex.org/C56461940","wikidata":"https://www.wikidata.org/wiki/Q970687","display_name":"Eye tracking","level":2,"score":0.5983449816703796},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.5630311965942383},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.543175220489502},{"id":"https://openalex.org/C2779916870","wikidata":"https://www.wikidata.org/wiki/Q14467155","display_name":"Gaze","level":2,"score":0.5312691926956177},{"id":"https://openalex.org/C146249460","wikidata":"https://www.wikidata.org/wiki/Q2914991","display_name":"Fixation (population genetics)","level":3,"score":0.5244922041893005},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5187435746192932},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4591890871524811},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.4388899505138397},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.42154625058174133},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3715474009513855},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3497515618801117},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.15919235348701477},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.12452250719070435},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.12336012721061707},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C2908647359","wikidata":"https://www.wikidata.org/wiki/Q2625603","display_name":"Population","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:1594025044","doi":null,"is_oa":false,"landing_page_url":"http://stp.lingfil.uu.se/~sara/papers/sltc10.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306420436","display_name":"Language and Technology Conference","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Language and Technology Conference","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W22168010","https://openalex.org/W314345615","https://openalex.org/W2101105183","https://openalex.org/W2103866953","https://openalex.org/W2124807415","https://openalex.org/W2133512280","https://openalex.org/W2463396630","https://openalex.org/W3198021593"],"related_works":["https://openalex.org/W2973065121","https://openalex.org/W2775584716","https://openalex.org/W3159892921","https://openalex.org/W7099030273","https://openalex.org/W7100075978","https://openalex.org/W2147192413","https://openalex.org/W2058941677","https://openalex.org/W2021618504","https://openalex.org/W2250613244","https://openalex.org/W249877898"],"abstract_inverted_index":{"Evaluation":[0],"of":[1,64,73,105,135,152,163,182,185,213,264],"machine":[2],"translation":[3,35,51],"(MT)":[4],"is":[5,25,61,101,168,206],"a":[6,50,54,164,278],"difficult":[7],"task,":[8],"both":[9,251],"for":[10,158,216,232,242,259,273],"humans,":[11],"and":[12,75,117,130,156,203,226,254],"using":[13,42,153,214],"automatic":[14,43,118],"metrics.":[15],"The":[16,98,262],"main":[17,99],"difficulty":[18],"lies":[19],"in":[20,237],"the":[21,140,143,186,207,222,270,288],"fact":[22],"that":[23,221,250],"there":[24],"not":[26,294],"one":[27],"single":[28,55],"correct":[29],"translation,":[30],"but":[31],"many":[32],"alternative":[33,60],"good":[34],"options.":[36],"MT":[37,96,159,178,217,260],"systems":[38,70,276,283],"are":[39,137,211,267],"often":[40,292],"evaluated":[41],"metrics,":[44,119],"which":[45],"commonly":[46,67],"rely":[47],"on":[48,77,90,102,127,198],"comparing":[49],"to":[52,120,175,179],"only":[53,208],"human":[56,65,106,114],"reference":[57],"translation.":[58],"An":[59],"different":[62,87,94,103,289],"types":[63,104],"evaluations,":[66],"ranking":[68],"between":[69,193,275],"or":[71,80],"estimations":[72],"adequacy":[74],"fluency":[76],"some":[78],"scale,":[79],"error":[81,115],"analyses.":[82],"We":[83,108],"have":[84,147],"explored":[85],"four":[86],"evaluation":[88,112,124,290],"methods":[89,125,134,266,272,291],"output":[91],"from":[92],"three":[93],"statistical":[95],"systems.":[97],"focus":[100],"evaluation.":[107,160,218,261],"compare":[109,176],"two":[110,121,133],"conventional":[111],"methods,":[113],"analysis":[116],"lesser":[122],"used":[123],"based":[126],"reading":[128,154,165,199,252],"comprehension":[129,155,166,200,253],"eyetracking.":[131],"These":[132],"evaluations":[136],"performed":[138],"without":[139],"subjects":[141],"seeing":[142],"source":[144],"sentence.":[145],"There":[146],"been":[148],"few":[149],"previous":[150,246],"attempts":[151],"eye-tracking":[157,215,255],"One":[161],"example":[162],"study":[167,209],"Fuji":[169],"(1999)":[170],"who":[171],"conducted":[172],"an":[173,238],"experiment":[174],"Englishto-Japanese":[177],"several":[180],"versions":[181],"manual":[183],"corrections":[184],"system":[187],"output.":[188],"He":[189],"found":[190,220],"significant":[191,297],"differences":[192,197],"texts":[194],"with":[195,269,277,284],"large":[196],"questions.":[201],"Doherty":[202],"O\u2019Brien":[204],"(2009)":[205],"we":[210,248],"aware":[212],"They":[219],"average":[223],"gaze":[224],"time":[225],"fixation":[227],"counts":[228],"were":[229],"significantly":[230],"lower":[231],"sentences":[233],"judged":[234],"as":[235],"excellent":[236],"earlier":[239],"evaluation,":[240],"than":[241],"bad":[243],"sentences.":[244],"Like":[245],"research":[247],"find":[249],"can":[256],"be":[257],"useful":[258],"results":[263],"these":[265],"consistent":[268],"other":[271],"comparison":[274],"big":[279],"quality":[280],"difference.":[281],"For":[282],"similar":[285],"quality,":[286],"however,":[287],"does":[293],"show":[295],"any":[296],"differences.":[298]},"counts_by_year":[{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2013,"cited_by_count":1}],"updated_date":"2026-09-24T07:57:37.273135","created_date":"2016-06-24T00:00:00"}