{"id":"https://openalex.org/W2963170148","doi":"https://doi.org/10.1017/s1351324919000275","title":"A survey of 25 years of evaluation","display_name":"A survey of 25 years of evaluation","publication_year":2019,"publication_date":"2019-07-19","ids":{"openalex":"https://openalex.org/W2963170148","doi":"https://doi.org/10.1017/s1351324919000275","mag":"2963170148"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324919000275","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324919000275","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/E4330FAEB9202EC490218E3220DDA291/S1351324919000275a.pdf/div-class-title-a-survey-of-25-years-of-evaluation-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/E4330FAEB9202EC490218E3220DDA291/S1351324919000275a.pdf/div-class-title-a-survey-of-25-years-of-evaluation-div.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016543371","display_name":"Kenneth Church","orcid":"https://orcid.org/0000-0001-8378-6069"},"institutions":[{"id":"https://openalex.org/I98301712","display_name":"Baidu (China)","ror":"https://ror.org/03vs3wt56","country_code":"CN","type":"company","lineage":["https://openalex.org/I98301712"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Kenneth Ward Church","raw_affiliation_strings":["Kenneth Ward Church, Baidu, Sunnyvale, CA 94089, USA","Kenneth Ward Church, Baidu, Sunnyvale, CA 94089, USA,"],"raw_orcid":"https://orcid.org/0000-0001-8378-6069","affiliations":[{"raw_affiliation_string":"Kenneth Ward Church, Baidu, Sunnyvale, CA 94089, USA","institution_ids":["https://openalex.org/I98301712"]},{"raw_affiliation_string":"Kenneth Ward Church, Baidu, Sunnyvale, CA 94089, USA,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5089825123","display_name":"Joel Hestness","orcid":"https://orcid.org/0000-0001-6920-0906"},"institutions":[{"id":"https://openalex.org/I4401726927","display_name":"Cerebras Systems (United States)","ror":"https://ror.org/040zz8080","country_code":null,"type":"company","lineage":["https://openalex.org/I4401726927"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Joel Hestness","raw_affiliation_strings":["Cerebras Systems, Los Altos, CA 94022, USA"],"raw_orcid":"https://orcid.org/0000-0001-6920-0906","affiliations":[{"raw_affiliation_string":"Cerebras Systems, Los Altos, CA 94022, USA","institution_ids":["https://openalex.org/I4401726927"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5016543371"],"corresponding_institution_ids":["https://openalex.org/I98301712"],"apc_list":null,"apc_paid":null,"fwci":1.1607,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.84293294,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"25","issue":"06","first_page":"753","last_page":"767"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9977999925613403,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7723076939582825},{"id":"https://openalex.org/keywords/sort","display_name":"sort","score":0.5648117065429688},{"id":"https://openalex.org/keywords/boom","display_name":"Boom","score":0.5193542242050171},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4999046325683594},{"id":"https://openalex.org/keywords/turing","display_name":"Turing","score":0.4826924800872803},{"id":"https://openalex.org/keywords/bust","display_name":"Bust","score":0.480089008808136},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.41876283288002014},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2302972674369812},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.15855759382247925}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7723076939582825},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.5648117065429688},{"id":"https://openalex.org/C141441539","wikidata":"https://www.wikidata.org/wiki/Q1970908","display_name":"Boom","level":2,"score":0.5193542242050171},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4999046325683594},{"id":"https://openalex.org/C9870796","wikidata":"https://www.wikidata.org/wiki/Q490481","display_name":"Turing","level":2,"score":0.4826924800872803},{"id":"https://openalex.org/C2778929795","wikidata":"https://www.wikidata.org/wiki/Q17489160","display_name":"Bust","level":3,"score":0.480089008808136},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.41876283288002014},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2302972674369812},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.15855759382247925},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C87717796","wikidata":"https://www.wikidata.org/wiki/Q146326","display_name":"Environmental engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s1351324919000275","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324919000275","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/E4330FAEB9202EC490218E3220DDA291/S1351324919000275a.pdf/div-class-title-a-survey-of-25-years-of-evaluation-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1017/s1351324919000275","is_oa":true,"landing_page_url":"https://doi.org/10.1017/s1351324919000275","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/E4330FAEB9202EC490218E3220DDA291/S1351324919000275a.pdf/div-class-title-a-survey-of-25-years-of-evaluation-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2963170148.pdf","grobid_xml":"https://content.openalex.org/works/W2963170148.grobid-xml"},"referenced_works_count":14,"referenced_works":["https://openalex.org/W1632114991","https://openalex.org/W1861492603","https://openalex.org/W1969655989","https://openalex.org/W2007709031","https://openalex.org/W2074504299","https://openalex.org/W2104621128","https://openalex.org/W2112796928","https://openalex.org/W2117539524","https://openalex.org/W2156985047","https://openalex.org/W2166637769","https://openalex.org/W2592634758","https://openalex.org/W2603004668","https://openalex.org/W2763406350","https://openalex.org/W2891288076"],"related_works":["https://openalex.org/W3130401151","https://openalex.org/W2480348369","https://openalex.org/W2964376215","https://openalex.org/W3122981517","https://openalex.org/W2288774045","https://openalex.org/W2944361826","https://openalex.org/W1519663516","https://openalex.org/W3089059013","https://openalex.org/W3030256362","https://openalex.org/W2115916304"],"abstract_inverted_index":{"Abstract":[0],"Evaluation":[1],"was":[2,10,19,29,68,98],"not":[3,212],"a":[4,11,33,36,99,110,142,187],"thing":[5,179,196],"when":[6],"the":[7,15,45,60,118,159,169,174,199,217],"first":[8,206],"author":[9],"graduate":[12],"student":[13],"in":[14,44,64,82],"late":[16],"1970s.":[17],"There":[18,139],"an":[20,181],"Artificial":[21],"Intelligence":[22],"(AI)":[23],"boom":[24,28],"then,":[25],"but":[26,152],"that":[27,71,96,209],"quickly":[30],"followed":[31],"by":[32,47],"bust":[34],"and":[35,76,84,88,131,135,149,167,191,193],"long":[37],"AI":[38],"Winter.":[39],"Charles":[40],"Wayne":[41],"restarted":[42],"funding":[43],"mid-1980s":[46],"emphasizing":[48],"evaluation.":[49],"No":[50],"other":[51,126],"sort":[52],"of":[53,117],"program":[54,67],"could":[55],"have":[56,78],"been":[57,141],"funded":[58],"at":[59,62],"time,":[61],"least":[63],"America.":[65],"His":[66],"so":[69],"successful":[70],"these":[72,155],"days,":[73],"shared":[74],"tasks":[75],"leaderboards":[77],"become":[79],"common":[80],"place":[81],"speech":[83],"language":[85],"(and":[86],"Vision":[87],"Machine":[89],"Learning).":[90],"It":[91,176],"is":[92,171,177,208],"hard":[93],"to":[94,146,164,183,185,197,220],"remember":[95],"evaluation":[97,182],"tough":[100],"sell":[101],"25":[102],"years":[103],"ago.":[104],"That":[105],"said,":[106,204],"we":[107],"may":[108],"be":[109,165],"bit":[111],"too":[112,162],"satisfied":[113],"with":[114],"current":[115],"state":[116],"art.":[119],"This":[120],"paper":[121],"will":[122],"survey":[123],"considerations":[124],"from":[125,133,137],"fields":[127],"such":[128],"as":[129],"reliability":[130],"validity":[132],"psychology":[134],"generalization":[136],"systems.":[138],"has":[140],"trend":[143],"for":[144,180],"publications":[145],"report":[147],"better":[148,150,172],"numbers,":[151],"what":[153],"do":[154],"numbers":[156,160],"mean?":[157],"Sometimes":[158],"are":[161,216],"good":[163],"true,":[166],"sometimes":[168],"truth":[170],"than":[173],"numbers.":[175],"one":[178],"fail":[184],"find":[186],"difference":[188],"between":[189],"man":[190],"machine,":[192],"quite":[194],"another":[195],"pass":[198],"Turing":[200],"Test.":[201],"As":[202],"Feynman":[203],"\u201cthe":[205],"principle":[207],"you":[210,215],"must":[211],"fool":[213],"yourself\u2013and":[214],"easiest":[218],"person":[219],"fool.\u201d":[221]},"counts_by_year":[{"year":2023,"cited_by_count":3},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}