{"id":"https://openalex.org/W2182894058","doi":"https://doi.org/10.1162/tacl_a_00167","title":"The Language Demographics of Amazon Mechanical Turk","display_name":"The Language Demographics of Amazon Mechanical Turk","publication_year":2014,"publication_date":"2014-12-01","ids":{"openalex":"https://openalex.org/W2182894058","doi":"https://doi.org/10.1162/tacl_a_00167","mag":"2182894058"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00167","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00167","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/tacl_a_00167","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/tacl_a_00167","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053850863","display_name":"Ellie Pavlick","orcid":"https://orcid.org/0000-0002-7155-5420"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ellie Pavlick","raw_affiliation_strings":["Computer and Information Science Department, University of                         Pennsylvania","Computer and Information Science Department, University of\n Pennsylvania"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer and Information Science Department, University of                         Pennsylvania","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Computer and Information Science Department, University of\n Pennsylvania","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108266978","display_name":"Matt Post","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I2802946424","display_name":"Johns Hopkins University Applied Physics Laboratory","ror":"https://ror.org/029pp9z10","country_code":"US","type":"facility","lineage":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Matt Post","raw_affiliation_strings":["Human Language Technology Center of Excellence, Johns Hopkins                         University","Human Language Technology Center of Excellence, Johns Hopkins\n University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins                         University","institution_ids":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]},{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins\n University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011118385","display_name":"Ann Irvine","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I2802946424","display_name":"Johns Hopkins University Applied Physics Laboratory","ror":"https://ror.org/029pp9z10","country_code":"US","type":"facility","lineage":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ann Irvine","raw_affiliation_strings":["Human Language Technology Center of Excellence, Johns Hopkins                         University","Human Language Technology Center of Excellence, Johns Hopkins\n University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins                         University","institution_ids":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]},{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins\n University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001423147","display_name":"Dmitry Kachaev","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I2802946424","display_name":"Johns Hopkins University Applied Physics Laboratory","ror":"https://ror.org/029pp9z10","country_code":"US","type":"facility","lineage":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dmitry Kachaev","raw_affiliation_strings":["Human Language Technology Center of Excellence, Johns Hopkins                         University","Human Language Technology Center of Excellence, Johns Hopkins\n University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins                         University","institution_ids":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]},{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins\n University","institution_ids":["https://openalex.org/I145311948"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068508539","display_name":"Chris Callison-Burch","orcid":"https://orcid.org/0000-0001-8196-1943"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I2802946424","display_name":"Johns Hopkins University Applied Physics Laboratory","ror":"https://ror.org/029pp9z10","country_code":"US","type":"facility","lineage":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]},{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Callison-Burch","raw_affiliation_strings":["Computer and Information Science Department, University of                         Pennsylvania","Human Language Technology Center of Excellence, Johns Hopkins                         University","Computer and Information Science Department, University of\n Pennsylvania","Human Language Technology Center of Excellence, Johns Hopkins\n University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computer and Information Science Department, University of                         Pennsylvania","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins                         University","institution_ids":["https://openalex.org/I145311948","https://openalex.org/I2802946424"]},{"raw_affiliation_string":"Computer and Information Science Department, University of\n Pennsylvania","institution_ids":["https://openalex.org/I79576946"]},{"raw_affiliation_string":"Human Language Technology Center of Excellence, Johns Hopkins\n University","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":15.0008,"has_fulltext":true,"cited_by_count":99,"citation_normalized_percentile":{"value":0.98758594,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"2","issue":null,"first_page":"79","last_page":"92"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.9628229141235352},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7994531393051147},{"id":"https://openalex.org/keywords/demographics","display_name":"Demographics","score":0.6080070734024048},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5786137580871582},{"id":"https://openalex.org/keywords/crowd-sourcing","display_name":"Crowd sourcing","score":0.5195463299751282},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.49375781416893005},{"id":"https://openalex.org/keywords/machine-translation","display_name":"Machine translation","score":0.4575374126434326},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4342220723628998},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3643496632575989},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3355181813240051},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07367032766342163}],"concepts":[{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.9628229141235352},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7994531393051147},{"id":"https://openalex.org/C2780084366","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demographics","level":2,"score":0.6080070734024048},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5786137580871582},{"id":"https://openalex.org/C3018396927","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowd sourcing","level":2,"score":0.5195463299751282},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.49375781416893005},{"id":"https://openalex.org/C203005215","wikidata":"https://www.wikidata.org/wiki/Q79798","display_name":"Machine translation","level":2,"score":0.4575374126434326},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4342220723628998},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3643496632575989},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3355181813240051},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07367032766342163},{"id":"https://openalex.org/C149923435","wikidata":"https://www.wikidata.org/wiki/Q37732","display_name":"Demography","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1162/tacl_a_00167","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00167","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/tacl_a_00167","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.681.9778","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.681.9778","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/viewFile/262/34/","raw_type":"text"},{"id":"pmh:oai:doaj.org/article:baf0a32a977c479abd6f0d7a811d8fbc","is_oa":false,"landing_page_url":"https://doaj.org/article/baf0a32a977c479abd6f0d7a811d8fbc","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 2 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00167","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00167","pdf_url":"http://www.mitpressjournals.org/doi/pdf/10.1162/tacl_a_00167","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.75}],"awards":[],"funders":[{"id":"https://openalex.org/F4320310145","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2182894058.pdf","grobid_xml":"https://content.openalex.org/works/W2182894058.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W137989762","https://openalex.org/W170711724","https://openalex.org/W191512969","https://openalex.org/W573259127","https://openalex.org/W628858056","https://openalex.org/W1522553155","https://openalex.org/W1577841485","https://openalex.org/W1724528691","https://openalex.org/W1785101966","https://openalex.org/W1808176545","https://openalex.org/W1947674219","https://openalex.org/W1959000896","https://openalex.org/W1970381522","https://openalex.org/W2022088830","https://openalex.org/W2058556535","https://openalex.org/W2081580037","https://openalex.org/W2097261017","https://openalex.org/W2102616831","https://openalex.org/W2103240268","https://openalex.org/W2104907655","https://openalex.org/W2114269021","https://openalex.org/W2115915304","https://openalex.org/W2119775030","https://openalex.org/W2127008633","https://openalex.org/W2127849236","https://openalex.org/W2133102719","https://openalex.org/W2143280940","https://openalex.org/W2149489787","https://openalex.org/W2161943765","https://openalex.org/W2170989440","https://openalex.org/W2182472765","https://openalex.org/W2185757438","https://openalex.org/W2194370442","https://openalex.org/W2251339511","https://openalex.org/W2294566554","https://openalex.org/W2523478734","https://openalex.org/W2733984477","https://openalex.org/W2913879771"],"related_works":["https://openalex.org/W2409650238","https://openalex.org/W2775554247","https://openalex.org/W2524552836","https://openalex.org/W2250395065","https://openalex.org/W2924423142","https://openalex.org/W2110168585","https://openalex.org/W3107474891","https://openalex.org/W2250213760","https://openalex.org/W4386247111","https://openalex.org/W4327642362"],"abstract_inverted_index":{"We":[0,17,37,141],"present":[1],"a":[2,19,74,87],"large":[3],"scale":[4],"study":[5,98,122],"of":[6,26,82,89,94,135,171],"the":[7,23,64,108,113,136,149,169],"languages":[8,65,147],"spoken":[9],"by":[10,44,53],"bilingual":[11,116,128,137,181],"workers":[12,29,55,117,178],"on":[13,107,139],"Mechanical":[14],"Turk":[15],"(MTurk).":[16],"establish":[18],"methodology":[20],"for":[21,100,156],"determining":[22],"language":[24,41],"skills":[25],"anonymous":[27],"crowd":[28],"that":[30,115],"is":[31],"more":[32],"robust":[33],"than":[34,72],"simple":[35],"surveying.":[36],"validate":[38],"workers\u2019":[39],"self-reported":[40],"skill":[42],"claims":[43],"measuring":[45],"their":[46],"ability":[47],"to":[48,56,68,85,126,131,145,161,179,191],"correctly":[49],"translate":[50,86],"words,":[51],"and":[52,103,130,188],"geolocating":[54],"see":[57],"if":[58],"they":[59],"reside":[60],"in":[61,92,184],"countries":[62],"where":[63],"are":[66],"likely":[67],"be":[69],"spoken.":[70],"Rather":[71],"posting":[73],"one-off":[75],"survey,":[76],"we":[77,176],"posted":[78],"paid":[79],"tasks":[80],"consisting":[81],"1,000":[83],"assignments":[84],"total":[88],"10,000":[90],"words":[91],"each":[93],"100":[95],"languages.":[96],"Our":[97,121],"ran":[99],"several":[101],"months,":[102],"was":[104,123],"highly":[105],"visible":[106],"MTurk":[109],"crowdsourcing":[110],"platform,":[111],"increasing":[112],"chances":[114],"would":[118],"complete":[119],"it.":[120],"useful":[124],"both":[125],"create":[127,180],"dictionaries":[129],"act":[132],"as":[133,153],"census":[134],"speakers":[138],"MTurk.":[140],"use":[142,189],"this":[143],"data":[144,173],"recommend":[146],"with":[148],"largest":[150],"speaker":[151],"populations":[152],"good":[154],"candidates":[155],"other":[157],"researchers":[158],"who":[159],"want":[160],"develop":[162],"crowdsourced,":[163],"multilingual":[164],"technologies.":[165],"To":[166],"further":[167],"demonstrate":[168],"value":[170],"creating":[172],"via":[174],"crowdsourcing,":[175],"hire":[177],"parallel":[182],"corpora":[183],"six":[185],"Indian":[186],"languages,":[187],"them":[190],"train":[192],"statistical":[193],"machine":[194],"translation":[195],"systems.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":10},{"year":2021,"cited_by_count":14},{"year":2020,"cited_by_count":7},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":5},{"year":2017,"cited_by_count":8},{"year":2016,"cited_by_count":11},{"year":2015,"cited_by_count":9},{"year":2014,"cited_by_count":5}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}