{"id":"https://openalex.org/W3212862835","doi":"https://doi.org/10.18653/v1/2021.eval4nlp-1.19","title":"Error-Sensitive Evaluation for Ordinal Target Variables","display_name":"Error-Sensitive Evaluation for Ordinal Target Variables","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3212862835","doi":"https://doi.org/10.18653/v1/2021.eval4nlp-1.19","mag":"3212862835"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2021.eval4nlp-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.eval4nlp-1.19","pdf_url":"https://aclanthology.org/2021.eval4nlp-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Evaluation and Comparison of NLP Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2021.eval4nlp-1.19.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100675683","display_name":"David Chen","orcid":"https://orcid.org/0009-0002-8386-7840"},"institutions":[{"id":"https://openalex.org/I1305444813","display_name":"Capital One (United States)","ror":"https://ror.org/00svp7168","country_code":"US","type":"company","lineage":["https://openalex.org/I1305444813"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Chen","raw_affiliation_strings":["Capital One, Vision and Language Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Capital One, Vision and Language Technologies","institution_ids":["https://openalex.org/I1305444813"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058589892","display_name":"Maury Courtland","orcid":null},"institutions":[{"id":"https://openalex.org/I1305444813","display_name":"Capital One (United States)","ror":"https://ror.org/00svp7168","country_code":"US","type":"company","lineage":["https://openalex.org/I1305444813"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Maury Courtland","raw_affiliation_strings":["Capital One, Vision and Language Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Capital One, Vision and Language Technologies","institution_ids":["https://openalex.org/I1305444813"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049364660","display_name":"Adam Faulkner","orcid":"https://orcid.org/0000-0002-3484-2041"},"institutions":[{"id":"https://openalex.org/I1305444813","display_name":"Capital One (United States)","ror":"https://ror.org/00svp7168","country_code":"US","type":"company","lineage":["https://openalex.org/I1305444813"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Adam Faulkner","raw_affiliation_strings":["Capital One, Vision and Language Technologies"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Capital One, Vision and Language Technologies","institution_ids":["https://openalex.org/I1305444813"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5057563896","display_name":"Aysu Ezen-Can","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aysu Ezen-Can","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.20655787,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"189","last_page":"199"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11918","display_name":"Forecasting Techniques and Applications","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10050","display_name":"Multi-Criteria Decision Making","score":0.9821000099182129,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9692999720573425,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/ordinal-data","display_name":"Ordinal data","score":0.5401983857154846},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5073053240776062},{"id":"https://openalex.org/keywords/ordinal-optimization","display_name":"Ordinal optimization","score":0.5005650520324707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4547227919101715},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.4402335286140442},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.397426038980484},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.33472681045532227}],"concepts":[{"id":"https://openalex.org/C85461838","wikidata":"https://www.wikidata.org/wiki/Q7100785","display_name":"Ordinal data","level":2,"score":0.5401983857154846},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5073053240776062},{"id":"https://openalex.org/C81386100","wikidata":"https://www.wikidata.org/wiki/Q7100792","display_name":"Ordinal optimization","level":3,"score":0.5005650520324707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4547227919101715},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.4402335286140442},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.397426038980484},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.33472681045532227}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.18653/v1/2021.eval4nlp-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.eval4nlp-1.19","pdf_url":"https://aclanthology.org/2021.eval4nlp-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Evaluation and Comparison of NLP Systems","raw_type":"proceedings-article"},{"id":"mag:3212862835","is_oa":false,"landing_page_url":"https://aclanthology.org/2021.eval4nlp-1.19/","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"doi:10.18653/v1/2021.eval4nlp-1.19","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2021.eval4nlp-1.19","pdf_url":"https://aclanthology.org/2021.eval4nlp-1.19.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2nd Workshop on Evaluation and Comparison of NLP Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3212862835.pdf","grobid_xml":"https://content.openalex.org/works/W3212862835.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W79937781","https://openalex.org/W1831460538","https://openalex.org/W1964841990","https://openalex.org/W1997855593","https://openalex.org/W2027608021","https://openalex.org/W2044538423","https://openalex.org/W2053509658","https://openalex.org/W2113459411","https://openalex.org/W2143747826","https://openalex.org/W2148143831","https://openalex.org/W2163455955","https://openalex.org/W2190044943","https://openalex.org/W2250730878","https://openalex.org/W2251939518","https://openalex.org/W2327678559","https://openalex.org/W2471350540","https://openalex.org/W2612395532","https://openalex.org/W2767106145","https://openalex.org/W2903757417","https://openalex.org/W2916132663","https://openalex.org/W2949547296","https://openalex.org/W2963023579","https://openalex.org/W2963205619","https://openalex.org/W2963341956","https://openalex.org/W2964230347","https://openalex.org/W2971014768","https://openalex.org/W2986379544","https://openalex.org/W3035729454","https://openalex.org/W3097760447","https://openalex.org/W3099215402","https://openalex.org/W3100880133","https://openalex.org/W3103379492","https://openalex.org/W3104604581"],"related_works":["https://openalex.org/W1984606470","https://openalex.org/W2798701209","https://openalex.org/W188694065","https://openalex.org/W201238850","https://openalex.org/W2496749787","https://openalex.org/W3048012572","https://openalex.org/W2788239595","https://openalex.org/W3038013251","https://openalex.org/W2081126853","https://openalex.org/W2067978978","https://openalex.org/W2982392333","https://openalex.org/W2410466099","https://openalex.org/W2793996608","https://openalex.org/W2084735613","https://openalex.org/W3175773141","https://openalex.org/W3182035769","https://openalex.org/W1797683501","https://openalex.org/W2044334607","https://openalex.org/W3096062020","https://openalex.org/W148648452"],"abstract_inverted_index":{"Product":[0],"reviews":[1],"and":[2,23,62,76,83,111,125,139,159],"satisfaction":[3],"surveys":[4],"seek":[5],"customer":[6],"feedback":[7],"in":[8,28,109,166],"the":[9,26,29,48,93,121,145],"form":[10],"of":[11,50],"ranked":[12],"scales.":[13],"In":[14,42],"these":[15],"settings,":[16],"widely":[17],"used":[18],"evaluation":[19],"metrics":[20,151],"including":[21],"F1":[22],"accuracy":[24,110],"ignore":[25],"rank":[27],"responses":[30],"(e.g.,":[31],"'very":[32],"likely'":[33],"is":[34,53,114],"closer":[35],"to":[36,85,147],"'likely'":[37],"than":[38,117],"'not":[39],"at":[40],"all').":[41],"this":[43,69],"paper,":[44],"we":[45,71,160],"hypothesize":[46],"that":[47,101,162],"order":[49],"class":[51],"values":[52],"important":[54],"for":[55,106,119,127,152,170],"evaluating":[56,153],"classifiers":[57],"on":[58],"ordinal":[59,89,156,171],"target":[60,90,126,157],"variables":[61,91],"should":[63],"not":[64],"be":[65],"disregarded.":[66],"To":[67],"test":[68],"hypothesis,":[70],"compared":[72],"Multi-class":[73],"Classification":[74],"(MC)":[75],"Ordinal":[77],"Regression":[78],"(OR)":[79],"by":[80,132],"applying":[81],"OR":[82,105,113],"MC":[84,103,118],"benchmark":[86],"tasks":[87],"involving":[88],"using":[92],"same":[94],"underlying":[95],"model":[96],"architecture.":[97],"Experimental":[98],"results":[99],"show":[100],"while":[102],"outperformed":[104],"some":[107],"datasets":[108],"F1,":[112],"significantly":[115],"better":[116],"minimizing":[120],"error":[122,137],"between":[123],"prediction":[124],"all":[128],"benchmarks,":[129],"as":[130],"revealed":[131],"error-sensitive":[133,150],"metrics,":[134],"e.g.":[135],"mean-squared":[136],"(MSE)":[138],"Spearman":[140],"correlation.":[141],"Our":[142],"findings":[143],"motivate":[144],"need":[146],"establish":[148],"consistent,":[149],"benchmarks":[154],"with":[155],"variables,":[158],"hope":[161],"it":[163],"stimulates":[164],"interest":[165],"exploring":[167],"alternative":[168],"losses":[169],"problems.":[172]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}