{"id":"https://openalex.org/W2566111950","doi":"https://doi.org/10.18653/v1/d16-1049","title":"IRT-based Aggregation Model of Crowdsourced Pairwise Comparison for Evaluating Machine Translations","display_name":"IRT-based Aggregation Model of Crowdsourced Pairwise Comparison for Evaluating Machine Translations","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2566111950","doi":"https://doi.org/10.18653/v1/d16-1049","mag":"2566111950"},"language":"en","primary_location":{"id":"doi:10.18653/v1/d16-1049","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1049","pdf_url":"https://www.aclweb.org/anthology/D16-1049.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/D16-1049.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103177065","display_name":"Naoki Otani","orcid":"https://orcid.org/0000-0002-5855-7616"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Naoki Otani","raw_affiliation_strings":["Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan"],"affiliations":[{"raw_affiliation_string":"Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070222190","display_name":"Toshiaki Nakazawa","orcid":null},"institutions":[{"id":"https://openalex.org/I4210086780","display_name":"Japan Science and Technology Agency","ror":"https://ror.org/00097mb19","country_code":"JP","type":"government","lineage":["https://openalex.org/I4210086780"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Toshiaki Nakazawa","raw_affiliation_strings":["Japan Science and Technology Agency, Kawaguchi-shi, Saitama, Japan"],"affiliations":[{"raw_affiliation_string":"Japan Science and Technology Agency, Kawaguchi-shi, Saitama, Japan","institution_ids":["https://openalex.org/I4210086780"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102867777","display_name":"Daisuke Kawahara","orcid":"https://orcid.org/0000-0002-3598-1027"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Daisuke Kawahara","raw_affiliation_strings":["Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan"],"affiliations":[{"raw_affiliation_string":"Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028836340","display_name":"Sadao Kurohashi","orcid":"https://orcid.org/0000-0001-5398-8399"},"institutions":[{"id":"https://openalex.org/I22299242","display_name":"Kyoto University","ror":"https://ror.org/02kpeqv85","country_code":"JP","type":"education","lineage":["https://openalex.org/I22299242"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sadao Kurohashi","raw_affiliation_strings":["Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan"],"affiliations":[{"raw_affiliation_string":"Graduate School of Informatics, Kyoto University, Yoshida-honmachi, Sakyo-ku, Kyoto, Japan","institution_ids":["https://openalex.org/I22299242"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5103177065"],"corresponding_institution_ids":["https://openalex.org/I22299242"],"apc_list":null,"apc_paid":null,"fwci":1.2854,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.87605216,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"511","last_page":"520"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.8741254806518555},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.8730289936065674},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.7947489023208618},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.728556752204895},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6046649813652039},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5983307361602783},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.520068347454071},{"id":"https://openalex.org/keywords/translation","display_name":"Translation (biology)","score":0.4864065647125244},{"id":"https://openalex.org/keywords/statistical-model","display_name":"Statistical model","score":0.485493928194046},{"id":"https://openalex.org/keywords/item-response-theory","display_name":"Item response theory","score":0.4549540877342224},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43359193205833435},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3449018597602844},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.202693372964859},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.13179776072502136},{"id":"https://openalex.org/keywords/psychometrics","display_name":"Psychometrics","score":0.06478860974311829}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.8741254806518555},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.8730289936065674},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.7947489023208618},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.728556752204895},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6046649813652039},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5983307361602783},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.520068347454071},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.4864065647125244},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.485493928194046},{"id":"https://openalex.org/C19875794","wikidata":"https://www.wikidata.org/wiki/Q1207340","display_name":"Item response theory","level":3,"score":0.4549540877342224},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43359193205833435},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3449018597602844},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.202693372964859},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.13179776072502136},{"id":"https://openalex.org/C171606756","wikidata":"https://www.wikidata.org/wiki/Q506132","display_name":"Psychometrics","level":2,"score":0.06478860974311829},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/d16-1049","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1049","pdf_url":"https://www.aclweb.org/anthology/D16-1049.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/d16-1049","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/d16-1049","pdf_url":"https://www.aclweb.org/anthology/D16-1049.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2016 Conference on Empirical Methods in Natural\n          Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2566111950.pdf","grobid_xml":"https://content.openalex.org/works/W2566111950.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W2000619651","https://openalex.org/W2100175927","https://openalex.org/W2115081467","https://openalex.org/W2123326341","https://openalex.org/W2153975459","https://openalex.org/W2158905336","https://openalex.org/W2164545125","https://openalex.org/W2250672026","https://openalex.org/W2251901824","https://openalex.org/W2251927615","https://openalex.org/W2251994258","https://openalex.org/W2260677151","https://openalex.org/W2270190199","https://openalex.org/W2294913621","https://openalex.org/W2295515094","https://openalex.org/W2315316408","https://openalex.org/W2763110165","https://openalex.org/W3003252051","https://openalex.org/W3146697253","https://openalex.org/W4254972346"],"related_works":["https://openalex.org/W3032998312","https://openalex.org/W135177976","https://openalex.org/W4384486036","https://openalex.org/W1503094549","https://openalex.org/W2337920774","https://openalex.org/W4286908577","https://openalex.org/W2886410948","https://openalex.org/W2025875869","https://openalex.org/W4318823662","https://openalex.org/W3207526114"],"abstract_inverted_index":{"Recent":[0],"work":[1],"on":[2,65,72],"machine":[3,40],"translation":[4,41],"has":[5],"used":[6],"crowdsourcing":[7],"to":[8,36],"reduce":[9],"costs":[10],"of":[11],"manual":[12,33],"evaluations.":[13],"However,":[14],"crowdsourced":[15],"judgments":[16],"are":[17],"often":[18],"biased":[19],"and":[20,77,87],"inaccurate.":[21],"In":[22],"this":[23],"paper,":[24],"we":[25],"present":[26],"a":[27,39,66],"statistical":[28],"model":[29,49],"that":[30,79],"aggregates":[31],"many":[32],"pairwise":[34],"comparisons":[35],"robustly":[37],"measure":[38],"system's":[42],"performance.":[43],"Our":[44],"method":[45],"applies":[46],"graded":[47],"response":[48,52],"from":[50,69],"item":[51],"theory":[53],"(IRT),":[54],"which":[55],"was":[56,88],"originally":[57],"developed":[58],"for":[59],"academic":[60],"tests.":[61],"We":[62],"conducted":[63],"experiments":[64],"public":[67],"dataset":[68],"the":[70],"Workshop":[71],"Statistical":[73],"Machine":[74],"Translation":[75],"2013,":[76],"found":[78],"our":[80],"approach":[81],"resulted":[82],"in":[83],"highly":[84],"interpretable":[85],"estimates":[86],"less":[89],"affected":[90],"by":[91],"noisy":[92],"judges":[93],"than":[94],"previously":[95],"proposed":[96],"methods.":[97]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}