{"id":"https://openalex.org/W2515910099","doi":"https://doi.org/10.18653/v1/w16-2501","title":"Intrinsic Evaluation of Word Vectors Fails to Predict Extrinsic Performance","display_name":"Intrinsic Evaluation of Word Vectors Fails to Predict Extrinsic Performance","publication_year":2016,"publication_date":"2016-01-01","ids":{"openalex":"https://openalex.org/W2515910099","doi":"https://doi.org/10.18653/v1/w16-2501","mag":"2515910099"},"language":"en","primary_location":{"id":"doi:10.18653/v1/w16-2501","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2501","pdf_url":"https://www.aclweb.org/anthology/W16-2501.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/W16-2501.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024945225","display_name":"Billy Chiu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]},{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB","KR"],"is_corresponding":true,"raw_author_name":"Billy Chiu","raw_affiliation_strings":["Language Technology Lab DTAL, University of Cambridge"],"affiliations":[{"raw_affiliation_string":"Language Technology Lab DTAL, University of Cambridge","institution_ids":["https://openalex.org/I4210107233","https://openalex.org/I241749"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081393566","display_name":"Anna Korhonen","orcid":"https://orcid.org/0000-0002-3692-3144"},"institutions":[{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]},{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]}],"countries":["GB","KR"],"is_corresponding":false,"raw_author_name":"Anna Korhonen","raw_affiliation_strings":["Language Technology Lab DTAL, University of Cambridge"],"affiliations":[{"raw_affiliation_string":"Language Technology Lab DTAL, University of Cambridge","institution_ids":["https://openalex.org/I4210107233","https://openalex.org/I241749"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5066925770","display_name":"Sampo Pyysalo","orcid":"https://orcid.org/0000-0002-6279-5000"},"institutions":[{"id":"https://openalex.org/I4210107233","display_name":"Language Science (South Korea)","ror":"https://ror.org/01h9v1373","country_code":"KR","type":"company","lineage":["https://openalex.org/I4210107233"]},{"id":"https://openalex.org/I241749","display_name":"University of Cambridge","ror":"https://ror.org/013meh722","country_code":"GB","type":"education","lineage":["https://openalex.org/I241749"]}],"countries":["GB","KR"],"is_corresponding":false,"raw_author_name":"Sampo Pyysalo","raw_affiliation_strings":["Language Technology Lab DTAL, University of Cambridge"],"affiliations":[{"raw_affiliation_string":"Language Technology Lab DTAL, University of Cambridge","institution_ids":["https://openalex.org/I4210107233","https://openalex.org/I241749"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5024945225"],"corresponding_institution_ids":["https://openalex.org/I241749","https://openalex.org/I4210107233"],"apc_list":null,"apc_paid":null,"fwci":24.7364,"has_fulltext":true,"cited_by_count":98,"citation_normalized_percentile":{"value":0.99490239,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.8096523880958557},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.770035982131958},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.7686015367507935},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6979447603225708},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6453200578689575},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6312225461006165},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.51409512758255},{"id":"https://openalex.org/keywords/downstream","display_name":"Downstream (manufacturing)","score":0.48308971524238586},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4734748601913452},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1336885392665863}],"concepts":[{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.8096523880958557},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.770035982131958},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.7686015367507935},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6979447603225708},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6453200578689575},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6312225461006165},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.51409512758255},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.48308971524238586},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4734748601913452},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1336885392665863},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/w16-2501","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2501","pdf_url":"https://www.aclweb.org/anthology/W16-2501.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/w16-2501","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/w16-2501","pdf_url":"https://www.aclweb.org/anthology/W16-2501.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 1st Workshop on Evaluating Vector-Space\n          Representations for NLP","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5699999928474426,"display_name":"No poverty","id":"https://metadata.un.org/sdg/1"}],"awards":[{"id":"https://openalex.org/G4293880762","display_name":null,"funder_award_id":"MR/M013049","funder_id":"https://openalex.org/F4320334626","funder_display_name":"Medical Research Council"},{"id":"https://openalex.org/G4498546091","display_name":"Literature-based discovery for cancer biology","funder_award_id":"MR/M013049/1","funder_id":"https://openalex.org/F4320334626","funder_display_name":"Medical Research Council"}],"funders":[{"id":"https://openalex.org/F4320334626","display_name":"Medical Research Council","ror":"https://ror.org/03x94j517"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2515910099.pdf","grobid_xml":"https://content.openalex.org/works/W2515910099.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W1576954243","https://openalex.org/W1614298861","https://openalex.org/W1615991656","https://openalex.org/W1632114991","https://openalex.org/W1854884267","https://openalex.org/W2026487812","https://openalex.org/W2053921957","https://openalex.org/W2067438047","https://openalex.org/W2098921539","https://openalex.org/W2103318667","https://openalex.org/W2117130368","https://openalex.org/W2132631284","https://openalex.org/W2137735870","https://openalex.org/W2142625445","https://openalex.org/W2143017621","https://openalex.org/W2144578941","https://openalex.org/W2153579005","https://openalex.org/W2158139315","https://openalex.org/W2158899491","https://openalex.org/W2170682101","https://openalex.org/W2251012068","https://openalex.org/W2251243116","https://openalex.org/W2251253014","https://openalex.org/W2251771443","https://openalex.org/W2251994258","https://openalex.org/W2252211741","https://openalex.org/W2257408573","https://openalex.org/W2515248967","https://openalex.org/W2611669587","https://openalex.org/W2951299559","https://openalex.org/W2952087486","https://openalex.org/W2952230511","https://openalex.org/W2953320089","https://openalex.org/W2964121744","https://openalex.org/W3099386342","https://openalex.org/W4235023933","https://openalex.org/W4294170691"],"related_works":["https://openalex.org/W1583765404","https://openalex.org/W4214653257","https://openalex.org/W2055438207","https://openalex.org/W2521424917","https://openalex.org/W3040203686","https://openalex.org/W4249524554","https://openalex.org/W2349021146","https://openalex.org/W35583307","https://openalex.org/W4398294854","https://openalex.org/W2436192316"],"abstract_inverted_index":{"The":[0],"quality":[1],"of":[2,13,27,56,64,77],"word":[3,14,41,57],"representations":[4,29],"is":[5],"frequently":[6],"assessed":[7],"using":[8,53],"correlation":[9,36],"with":[10],"human":[11],"judgements":[12],"similarity.":[15],"Here,":[16],"we":[17],"question":[18],"whether":[19],"such":[20],"intrinsic":[21,72,100],"evaluation":[22,101,106],"can":[23,85],"predict":[24],"the":[25,28,35],"merits":[26],"for":[30],"downstream":[31,78],"tasks.":[32],"We":[33,80,103],"study":[34],"between":[37],"results":[38],"on":[39,47],"ten":[40],"similarity":[42,96],"benchmarks":[43],"and":[44,68],"tagger":[45],"performance":[46],"three":[48],"standard":[49],"sequence":[50],"labeling":[51],"tasks":[52],"a":[54,91],"variety":[55],"vectors":[58],"induced":[59],"from":[60,97],"an":[61],"unannotated":[62],"corpus":[63],"3.8":[65],"billion":[66],"words,":[67],"demonstrate":[69],"that":[70,82],"most":[71],"evaluations":[73],"are":[74],"poor":[75],"predictors":[76],"performance.":[79],"argue":[81],"this":[83],"issue":[84],"be":[86],"traced":[87],"in":[88,99],"part":[89],"to":[90,93,110],"failure":[92],"distinguish":[94],"specific":[95],"relatedness":[98],"datasets.":[102],"make":[104],"our":[105],"tools":[107],"openly":[108],"available":[109],"facilitate":[111],"further":[112],"study.":[113]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":17},{"year":2019,"cited_by_count":15},{"year":2018,"cited_by_count":24},{"year":2017,"cited_by_count":10},{"year":2016,"cited_by_count":7}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}