{"id":"https://openalex.org/W2159107349","doi":null,"title":"Findings of the 2010 Joint Workshop on Statistical Machine Translation and Metrics for Machine Translation","display_name":"Findings of the 2010 Joint Workshop on Statistical Machine Translation and Metrics for Machine Translation","publication_year":2010,"publication_date":"2010-01-01","ids":{"openalex":"https://openalex.org/W2159107349","mag":"2159107349"},"language":"en","primary_location":{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/365e5f89-fd3a-4bbf-beac-76b9d387a3c7","doi":null,"is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/findings-of-the-2010-joint-workshop-on-statistical-machine-translation-and-metrics-for-machine-translation(365e5f89-fd3a-4bbf-beac-76b9d387a3c7).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://www.aclweb.org/anthology/W/W10/W10-1703.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068508539","display_name":"Chris Callison-Burch","orcid":"https://orcid.org/0000-0001-8196-1943","observed_orcids":["https://orcid.org/0000-0001-8196-1943"]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Callison-Burch, C.","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112315093","display_name":"Philipp Koehn","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Koehn, P.","raw_affiliation_strings":["University of Edinburgh,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Edinburgh,","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109059955","display_name":"Christof Monz","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Monz, C.","raw_affiliation_strings":["University of Amsterdam"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Amsterdam","institution_ids":["https://openalex.org/I887064364"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031281793","display_name":"Kay Peterson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"government","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peterson, K.","raw_affiliation_strings":["National Institute of Standards and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Standards and Technology","institution_ids":["https://openalex.org/I1321296531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061045207","display_name":"Mark A. Przybocki","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1321296531","display_name":"National Institute of Standards and Technology","ror":"https://ror.org/05xpvk416","country_code":"US","type":"government","lineage":["https://openalex.org/I1321296531","https://openalex.org/I1343035065"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Przybocki, M.","raw_affiliation_strings":["National Institute of Standards and Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Standards and Technology","institution_ids":["https://openalex.org/I1321296531"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089575055","display_name":"Omar F. Zaidan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zaidan, O.F.","raw_affiliation_strings":["Johns Hopkins University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":29.5868,"has_fulltext":true,"cited_by_count":178,"citation_normalized_percentile":{"value":0.99908862,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":100},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10759","display_name":"Translation Studies and Practices","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1203","display_name":"Language and Linguistics"},"field":{"id":"https://openalex.org/fields/12","display_name":"Arts and Humanities"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10201","display_name":"Speech Recognition and Synthesis","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/machine-translation","display_name":"machine translation","score":0.6549999713897705},{"id":"https://openalex.org/keywords/machine-translation-evaluation","display_name":"machine translation evaluation","score":0.593999981880188},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.7459999918937683},{"id":"https://openalex.org/keywords/system-combination","display_name":"system combination","score":0.6790000200271606},{"id":"https://openalex.org/keywords/shared-task","display_name":"shared task","score":0.6399999856948853},{"id":"https://openalex.org/keywords/non-expert-annotators","display_name":"non-expert annotators","score":0.8209999799728394},{"id":"https://openalex.org/keywords/amazon-mechanical-turk","display_name":"Amazon Mechanical Turk","score":0.7770000100135803}],"concepts":[{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.8884235620498657},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8031320571899414},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7094486355781555},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6569540500640869},{"id":"https://openalex.org/C135784402","wikidata":"https://www.wikidata.org/wiki/Q6958279","display_name":"Evaluation of machine translation","level":5,"score":0.6269125938415527},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.5824267864227295},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.566016674041748},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5583086609840393},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4925139844417572},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.46245765686035156},{"id":"https://openalex.org/C146047270","wikidata":"https://www.wikidata.org/wiki/Q469666","display_name":"Human\u2013machine system","level":2,"score":0.45414942502975464},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4539102017879486},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4494313597679138},{"id":"https://openalex.org/C148526163","wikidata":"https://www.wikidata.org/wiki/Q6723733","display_name":"Machine translation software usability","level":4,"score":0.43421292304992676},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.4179893732070923},{"id":"https://openalex.org/C2986258472","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation system","level":3,"score":0.4159933924674988},{"id":"https://openalex.org/C24687705","wikidata":"https://www.wikidata.org/wiki/Q3753284","display_name":"Example-based machine translation","level":3,"score":0.3009066581726074},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2562979459762573},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0718262791633606},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.05145364999771118},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C170154142","wikidata":"https://www.wikidata.org/wiki/Q150737","display_name":"Architectural engineering","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C105580179","wikidata":"https://www.wikidata.org/wiki/Q188928","display_name":"Messenger RNA","level":3,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":7,"locations":[{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/365e5f89-fd3a-4bbf-beac-76b9d387a3c7","doi":null,"is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/findings-of-the-2010-joint-workshop-on-statistical-machine-translation-and-metrics-for-machine-translation(365e5f89-fd3a-4bbf-beac-76b9d387a3c7).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:dare.uva.nl:publications/365e5f89-fd3a-4bbf-beac-76b9d387a3c7","doi":null,"is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/findings-of-the-2010-joint-workshop-on-statistical-machine-translation-and-metrics-for-machine-translation(365e5f89-fd3a-4bbf-beac-76b9d387a3c7).html","pdf_url":"http://www.aclweb.org/anthology/W/W10/W10-1703.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.170.7957","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.170.7957","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://aclweb.org/anthology-new/W/W10/W10-1703.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.174.7361","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.174.7361","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www-csli.stanford.edu/%7Eccb/publications/findings-of-wmt10-and-metrics-matr.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.381.4054","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.381.4054","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cs.jhu.edu/~ccb/publications/findings-of-the-wmt10-shared-tasks.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.678.920","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.678.920","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ilps.science.uva.nl/sites/ilps.science.uva.nl/files/wmt10_1.pdf","raw_type":"text"},{"id":"pmh:uvapub:oai:dare.uva.nl:publications/365e5f89-fd3a-4bbf-beac-76b9d387a3c7","doi":null,"is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/findings-of-the-2010-joint-workshop-on-statistical-machine-translation-and-metrics-for-machine-translation(365e5f89-fd3a-4bbf-beac-76b9d387a3c7).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Joint Fifth Workshop on Statistical Machine Translation and MetricsMATR: ACL 2010: proceedings of the workshop: 15-16 July 2010, Uppsale University, Uppsala, Sweden, 17 - 53","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"pmh:oai:dare.uva.nl:publications/365e5f89-fd3a-4bbf-beac-76b9d387a3c7","doi":null,"is_oa":true,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/findings-of-the-2010-joint-workshop-on-statistical-machine-translation-and-metrics-for-machine-translation(365e5f89-fd3a-4bbf-beac-76b9d387a3c7).html","pdf_url":"http://www.aclweb.org/anthology/W/W10/W10-1703.pdf","source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G467554628","display_name":"RI: Multi-Level Modeling of Language and Translation","funder_award_id":"0713448","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2159107349.pdf","grobid_xml":"https://content.openalex.org/works/W2159107349.grobid-xml"},"referenced_works_count":58,"referenced_works":["https://openalex.org/W43934831","https://openalex.org/W141243353","https://openalex.org/W165422532","https://openalex.org/W193080678","https://openalex.org/W203061456","https://openalex.org/W749819666","https://openalex.org/W1491049622","https://openalex.org/W1500152046","https://openalex.org/W1500281234","https://openalex.org/W1506572002","https://openalex.org/W1510632846","https://openalex.org/W1525066359","https://openalex.org/W1536589074","https://openalex.org/W1537121970","https://openalex.org/W1545286665","https://openalex.org/W1561066061","https://openalex.org/W1584791343","https://openalex.org/W1594398153","https://openalex.org/W1644288249","https://openalex.org/W1650129375","https://openalex.org/W1663287312","https://openalex.org/W1834061219","https://openalex.org/W1854082259","https://openalex.org/W1893654245","https://openalex.org/W1896570304","https://openalex.org/W1899181831","https://openalex.org/W1950756615","https://openalex.org/W2060833990","https://openalex.org/W2061910127","https://openalex.org/W2100183594","https://openalex.org/W2115081467","https://openalex.org/W2119741585","https://openalex.org/W2124807415","https://openalex.org/W2125504239","https://openalex.org/W2127849236","https://openalex.org/W2133845434","https://openalex.org/W2137550424","https://openalex.org/W2143539737","https://openalex.org/W2147192413","https://openalex.org/W2147913166","https://openalex.org/W2149327368","https://openalex.org/W2149772987","https://openalex.org/W2149811020","https://openalex.org/W2152311128","https://openalex.org/W2157775952","https://openalex.org/W2161694750","https://openalex.org/W2162499915","https://openalex.org/W2164777277","https://openalex.org/W2169279899","https://openalex.org/W2170233162","https://openalex.org/W2172253183","https://openalex.org/W2251194847","https://openalex.org/W2251782722","https://openalex.org/W2252110609","https://openalex.org/W2571124179","https://openalex.org/W2916467912","https://openalex.org/W2917029047","https://openalex.org/W2917066545"],"related_works":["https://openalex.org/W2087735403","https://openalex.org/W7098497333","https://openalex.org/W7095945111","https://openalex.org/W7209380371","https://openalex.org/W2147192413","https://openalex.org/W2169279899","https://openalex.org/W2115081467","https://openalex.org/W3011381785","https://openalex.org/W3186081172","https://openalex.org/W95780973"],"abstract_inverted_index":{"This":[0,64],"paper":[1],"presents":[2],"the":[3,6,43,70],"results":[4],"of":[5,31,45,58,72],"WMT10":[7],"and":[8,21,36],"MetricsMATR10":[9],"shared":[10],"tasks,":[11],"which":[12],"included":[13],"a":[14,17,27],"translation":[15,34,59],"task,":[16,20],"system":[18,38],"combination":[19,39],"an":[22],"evaluation":[23,30],"task.":[24],"We":[25,41],"conducted":[26],"large-scale":[28],"manual":[29],"104":[32],"machine":[33],"systems":[35,47],"41":[37],"entries.":[40],"used":[42],"ranking":[44],"these":[46],"to":[48],"measure":[49],"how":[50],"strongly":[51],"automatic":[52],"metrics":[53],"correlate":[54],"with":[55],"human":[56,73],"judgments":[57,74],"quality":[60],"for":[61],"26":[62],"metrics.":[63],"year":[65],"we":[66],"also":[67],"investigated":[68],"increasing":[69],"number":[71],"by":[75],"hiring":[76],"non-expert":[77],"annotators":[78],"through":[79],"Amazon\u2019s":[80],"Mechanical":[81],"Turk.":[82]},"counts_by_year":[{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":10},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":14},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":15},{"year":2013,"cited_by_count":21},{"year":2012,"cited_by_count":40}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}