{"id":"https://openalex.org/W2387546565","doi":"https://doi.org/10.18653/v1/w16-2506","title":"Problems With Evaluation of Word Embeddings Using Word Similarity Tasks","display_name":"Problems With Evaluation of Word Embeddings Using Word Similarity Tasks","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2387546565","doi":"https://doi.org/10.18653/v1/w16-2506","mag":"2387546565"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w16-2506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2506","pdf_url":"https://www.aclweb.org/anthology/W16-2506.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W16-2506.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014855689","display_name":"Manaal Faruqui","orcid":null},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Manaal Faruqui","raw_affiliation_strings":["Language Technologies Institute, Carnegie Mellon University","Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062910836","display_name":"Yulia Tsvetkov","orcid":"https://orcid.org/0000-0002-4634-7128"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yulia Tsvetkov","raw_affiliation_strings":["Language Technologies Institute, Carnegie Mellon University","Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015147112","display_name":"Pushpendre Rastogi","orcid":"https://orcid.org/0000-0003-1573-3066"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pushpendre Rastogi","raw_affiliation_strings":["Department of Computer Science, Johns Hopkins University","Carnegie Mellon University"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]},{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5111222692","display_name":"Chris Dyer","orcid":null},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]},{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chris Dyer","raw_affiliation_strings":["Language Technologies Institute, Carnegie Mellon University","Johns Hopkins University"],"affiliations":[{"raw_affiliation_string":"Language Technologies Institute, Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]},{"raw_affiliation_string":"Johns Hopkins University","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5014855689"],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":15.0641,"has_fulltext":true,"cited_by_count":53,"citation_normalized_percentile":{"value":0.98894165,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"30","last_page":"35"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9968000054359436,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.8457227349281311},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.7797904014587402},{"id":"https://openalex.org/keywords/semantic-similarity","display_name":"Semantic similarity","score":0.70367431640625},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6962215900421143},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6958319544792175},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6345709562301636},{"id":"https://openalex.org/keywords/semeval","display_name":"SemEval","score":0.452658087015152},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.428541898727417},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.16274407505989075},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.14484474062919617},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.07507351040840149}],"concepts":[{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.8457227349281311},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.7797904014587402},{"id":"https://openalex.org/C130318100","wikidata":"https://www.wikidata.org/wiki/Q2268914","display_name":"Semantic similarity","level":2,"score":0.70367431640625},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6962215900421143},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6958319544792175},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6345709562301636},{"id":"https://openalex.org/C44572571","wikidata":"https://www.wikidata.org/wiki/Q7448970","display_name":"SemEval","level":3,"score":0.452658087015152},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.428541898727417},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.16274407505989075},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.14484474062919617},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.07507351040840149},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.18653/v1/w16-2506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2506","pdf_url":"https://www.aclweb.org/anthology/W16-2506.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1605.02276","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1605.02276","pdf_url":"https://arxiv.org/pdf/1605.02276","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.1605.02276","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1605.02276","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"},{"id":"doi:10.57702/yfpq8zb2","is_oa":true,"landing_page_url":"https://doi.org/10.57702/yfpq8zb2","pdf_url":null,"source":{"id":"https://openalex.org/S7407053387","display_name":"TIB Data Manager","issn_l":null,"issn":[],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"dataset"},{"id":"mag:2387546565","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"doi:10.18653/v1/w16-2506","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2506","pdf_url":"https://www.aclweb.org/anthology/W16-2506.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Responsible consumption and production","id":"https://metadata.un.org/sdg/12","score":0.4399999976158142}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2387546565.pdf","grobid_xml":"https://content.openalex.org/works/W2387546565.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W342285082","https://openalex.org/W355000591","https://openalex.org/W1574901103","https://openalex.org/W1615991656","https://openalex.org/W2026487812","https://openalex.org/W2036181987","https://openalex.org/W2053921957","https://openalex.org/W2080100102","https://openalex.org/W2097137621","https://openalex.org/W2103318667","https://openalex.org/W2111216449","https://openalex.org/W2117130368","https://openalex.org/W2125031621","https://openalex.org/W2125786288","https://openalex.org/W2132631284","https://openalex.org/W2137735870","https://openalex.org/W2141599568","https://openalex.org/W2142625445","https://openalex.org/W2144600658","https://openalex.org/W2158139315","https://openalex.org/W2162456950","https://openalex.org/W2164019165","https://openalex.org/W2164973920","https://openalex.org/W2170682101","https://openalex.org/W2176085882","https://openalex.org/W2250189634","https://openalex.org/W2250508463","https://openalex.org/W2250539671","https://openalex.org/W2251012068","https://openalex.org/W2251066368","https://openalex.org/W2252211741","https://openalex.org/W2289758118","https://openalex.org/W2294979170","https://openalex.org/W2295584157","https://openalex.org/W2882319491","https://openalex.org/W2950577311","https://openalex.org/W2951683451","https://openalex.org/W2963061446"],"related_works":["https://openalex.org/W2963419157","https://openalex.org/W2250539671","https://openalex.org/W2950577311","https://openalex.org/W1854884267","https://openalex.org/W2950133940","https://openalex.org/W2882319491","https://openalex.org/W2493916176","https://openalex.org/W2132339004","https://openalex.org/W2170682101","https://openalex.org/W2137735870","https://openalex.org/W2252211741","https://openalex.org/W168564468","https://openalex.org/W2112184938","https://openalex.org/W2053921957","https://openalex.org/W2251771443","https://openalex.org/W2251012068","https://openalex.org/W2141599568","https://openalex.org/W2125031621","https://openalex.org/W2117130368","https://openalex.org/W2381396677"],"abstract_inverted_index":{"Lacking":[0],"standardized":[1],"extrinsic":[2],"evaluation":[3,25,63,86,99],"methods":[4],"for":[5,23,85,95],"vector":[6],"representations":[7],"of":[8,26,41,64,81,87],"words,":[9],"the":[10,34,62,79],"NLP":[11],"community":[12],"has":[13],"relied":[14],"heavily":[15],"on":[16,67,98],"word":[17,27,65,68,82,88],"similarity":[18,30,69,83],"tasks":[19,84],"as":[20],"a":[21],"proxy":[22],"intrinsic":[24],"vectors.":[28],"Word":[29],"evaluation,":[31],"which":[32],"correlates":[33],"distance":[35],"between":[36],"vectors":[37,66,89],"and":[38,51,71,93],"human":[39],"judgments":[40],"\"semantic":[42],"similarity\"":[43],"is":[44,48,90],"attractive,":[45],"because":[46],"it":[47],"computationally":[49],"inexpensive":[50],"fast.":[52],"In":[53],"this":[54],"paper":[55],"we":[56],"present":[57],"several":[58],"problems":[59],"associated":[60],"with":[61],"datasets,":[70],"summarize":[72],"existing":[73],"solutions.":[74],"Our":[75],"study":[76],"suggests":[77],"that":[78],"use":[80],"not":[91],"sustainable":[92],"calls":[94],"further":[96],"research":[97],"methods.":[100]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2021,"cited_by_count":5},{"year":2020,"cited_by_count":11},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":10},{"year":2017,"cited_by_count":11},{"year":2016,"cited_by_count":4}],"updated_date":"2026-03-07T13:37:22.277990","created_date":"2025-10-10T00:00:00"}