{"id":"https://openalex.org/W4396242195","doi":"https://doi.org/10.1162/tacl_a_00675","title":"A Closer Look at Classification Evaluation Metrics and a Critical Reflection of Common Evaluation Practice","display_name":"A Closer Look at Classification Evaluation Metrics and a Critical Reflection of Common Evaluation Practice","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4396242195","doi":"https://doi.org/10.1162/tacl_a_00675"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00675","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00675","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00675","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010534476","display_name":"Juri Opitz","orcid":null},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Juri Opitz","raw_affiliation_strings":["University of Zurich, Switzerland. opitz.sci@gmail.com"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Zurich, Switzerland. opitz.sci@gmail.com","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5010534476"],"corresponding_institution_ids":["https://openalex.org/I202697423"],"apc_list":null,"apc_paid":null,"fwci":44.5324,"has_fulltext":true,"cited_by_count":131,"citation_normalized_percentile":{"value":0.99938733,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"12","issue":null,"first_page":"820","last_page":"836"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12010","display_name":"Evaluation and Performance Assessment","score":0.23829999566078186,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12010","display_name":"Evaluation and Performance Assessment","score":0.23829999566078186,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reflection","display_name":"Reflection (computer programming)","score":0.6092671751976013},{"id":"https://openalex.org/keywords/critical-reflection","display_name":"Critical reflection","score":0.5551443696022034},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.379853218793869},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.357387900352478},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.35011929273605347},{"id":"https://openalex.org/keywords/epistemology","display_name":"Epistemology","score":0.3322058916091919},{"id":"https://openalex.org/keywords/philosophy","display_name":"Philosophy","score":0.1728718876838684},{"id":"https://openalex.org/keywords/pedagogy","display_name":"Pedagogy","score":0.12111470103263855},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.05959710478782654}],"concepts":[{"id":"https://openalex.org/C65682993","wikidata":"https://www.wikidata.org/wiki/Q1056451","display_name":"Reflection (computer programming)","level":2,"score":0.6092671751976013},{"id":"https://openalex.org/C2986913963","wikidata":"https://www.wikidata.org/wiki/Q843894","display_name":"Critical reflection","level":2,"score":0.5551443696022034},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.379853218793869},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.357387900352478},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35011929273605347},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.3322058916091919},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.1728718876838684},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.12111470103263855},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.05959710478782654}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1162/tacl_a_00675","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00675","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2404.16958","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2404.16958","pdf_url":"https://arxiv.org/pdf/2404.16958","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:doi:10.5167/uzh-261471","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:doaj.org/article:1bc107b196c04911a265ffe3ef420b8b","is_oa":true,"landing_page_url":"https://doaj.org/article/1bc107b196c04911a265ffe3ef420b8b","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00675","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00675","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W1994359007","https://openalex.org/W2014364160","https://openalex.org/W2053154970","https://openalex.org/W2059185913","https://openalex.org/W2115610681","https://openalex.org/W2158698691","https://openalex.org/W2170505850","https://openalex.org/W2343649478","https://openalex.org/W2612742774","https://openalex.org/W2750618026","https://openalex.org/W2750747353","https://openalex.org/W2750880011","https://openalex.org/W2752647243","https://openalex.org/W2753523903","https://openalex.org/W2753768434","https://openalex.org/W2806198715","https://openalex.org/W2807333695","https://openalex.org/W2888728157","https://openalex.org/W2916132663","https://openalex.org/W2918408501","https://openalex.org/W2922580172","https://openalex.org/W2963119602","https://openalex.org/W2963875483","https://openalex.org/W2964094366","https://openalex.org/W2975256032","https://openalex.org/W2996044589","https://openalex.org/W2999309192","https://openalex.org/W3034984188","https://openalex.org/W3035507081","https://openalex.org/W3098886914","https://openalex.org/W3099606977","https://openalex.org/W3115467802","https://openalex.org/W3183569911","https://openalex.org/W4287855110","https://openalex.org/W4385571125","https://openalex.org/W4389518984","https://openalex.org/W6639295645","https://openalex.org/W6639583693","https://openalex.org/W6687403153","https://openalex.org/W6756816197","https://openalex.org/W6770844057","https://openalex.org/W6778942181","https://openalex.org/W6781905506","https://openalex.org/W6803717939","https://openalex.org/W6903740755"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2931662336","https://openalex.org/W2077865380","https://openalex.org/W4211124957","https://openalex.org/W2052149723","https://openalex.org/W2580144469","https://openalex.org/W608560531","https://openalex.org/W2355866088","https://openalex.org/W1607061251"],"abstract_inverted_index":{"Abstract":[0],"Classification":[1],"systems":[2,41],"are":[3,22],"evaluated":[4],"in":[5,74,115,124,131],"a":[6,56,64,152],"countless":[7],"number":[8],"of":[9,85,93],"papers.":[10,116],"However,":[11],"we":[12,89,118],"find":[13,136],"that":[14,137,149],"evaluation":[15,95,129],"practice":[16,130],"is":[17,60,140],"often":[18,141],"nebulous.":[19],"Frequently,":[20],"metrics":[21,38],"selected":[23],"without":[24],"arguments,":[25,146],"and":[26,70,87,106,127,163,168],"blurry":[27],"terminology":[28],"invites":[29],"misconceptions.":[30],"For":[31],"instance,":[32],"many":[33],"works":[34],"use":[35],"so-called":[36],"\u2018macro\u2019":[37,57],"to":[39],"rank":[40],"(e.g.,":[42],"\u2018macro":[43],"F1\u2019)":[44],"but":[45],"do":[46],"not":[47,142],"clearly":[48],"specify":[49],"what":[50],"they":[51,108],"would":[52],"expect":[53],"from":[54,81],"such":[55],"metric.":[58],"This":[59],"problematic,":[61],"since":[62],"picking":[63],"metric":[65,138,170],"can":[66,150],"affect":[67],"research":[68],"findings":[69],"thus":[71],"any":[72],"clarity":[73],"the":[75,82,102,121,125],"process":[76],"should":[77],"be":[78],"maximized.":[79],"Starting":[80],"intuitive":[83],"concepts":[84],"bias":[86],"prevalence,":[88],"perform":[90],"an":[91,147],"analysis":[92,98],"common":[94],"metrics.":[96],"The":[97],"helps":[99],"us":[100],"understand":[101],"metrics\u2019":[103],"underlying":[104],"properties,":[105],"how":[107],"align":[109],"with":[110,144],"expectations":[111],"as":[112],"found":[113],"expressed":[114],"Then":[117],"reflect":[119],"on":[120],"practical":[122],"situation":[123],"field,":[126],"survey":[128],"recent":[132],"shared":[133],"tasks.":[134],"We":[135],"selection":[139],"supported":[143],"convincing":[145],"issue":[148],"make":[151],"system":[153],"ranking":[154],"seem":[155],"arbitrary.":[156],"Our":[157],"work":[158],"aims":[159],"at":[160],"providing":[161],"overview":[162],"guidance":[164],"for":[165],"more":[166],"informed":[167],"transparent":[169],"selection,":[171],"fostering":[172],"meaningful":[173],"evaluation.":[174]},"counts_by_year":[{"year":2026,"cited_by_count":64},{"year":2025,"cited_by_count":63},{"year":2024,"cited_by_count":4}],"updated_date":"2026-08-24T07:32:12.397491","created_date":"2025-10-10T00:00:00"}