{"id":"https://openalex.org/W2734639922","doi":"https://doi.org/10.1145/3091478.3098871","title":"The Limits of Abstract Evaluation Metrics","display_name":"The Limits of Abstract Evaluation Metrics","publication_year":2017,"publication_date":"2017-06-25","ids":{"openalex":"https://openalex.org/W2734639922","doi":"https://doi.org/10.1145/3091478.3098871","mag":"2734639922"},"language":"en","primary_location":{"id":"doi:10.1145/3091478.3098871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3091478.3098871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM on Web Science Conference","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5103216912","display_name":"Alexandra Olteanu","orcid":"https://orcid.org/0000-0001-5710-4511"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Alexandra Olteanu","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060691179","display_name":"Kartik Talamadupula","orcid":"https://orcid.org/0000-0002-4628-3785"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kartik Talamadupula","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5015286159","display_name":"Kush R. Varshney","orcid":"https://orcid.org/0000-0002-7376-5536"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kush R. Varshney","raw_affiliation_strings":["IBM Research, Yorktown Heights, NY, USA"],"affiliations":[{"raw_affiliation_string":"IBM Research, Yorktown Heights, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5103216912"],"corresponding_institution_ids":["https://openalex.org/I1341412227"],"apc_list":null,"apc_paid":null,"fwci":2.4862,"has_fulltext":false,"cited_by_count":30,"citation_normalized_percentile":{"value":0.91674898,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"405","last_page":"406"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12262","display_name":"Hate Speech and Cyberbullying Detection","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11147","display_name":"Misinformation and Its Impacts","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/3312","display_name":"Sociology and Political Science"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.8699281215667725},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7708362340927124},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.6986192464828491},{"id":"https://openalex.org/keywords/perception","display_name":"Perception","score":0.620251476764679},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5919881463050842},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5613855123519897},{"id":"https://openalex.org/keywords/inclusion","display_name":"Inclusion (mineral)","score":0.4375263452529907},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.39424604177474976},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.36403143405914307},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.34059035778045654},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.22908064723014832},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.10036230087280273},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09556907415390015},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08429700136184692}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.8699281215667725},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7708362340927124},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.6986192464828491},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.620251476764679},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5919881463050842},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5613855123519897},{"id":"https://openalex.org/C109359841","wikidata":"https://www.wikidata.org/wiki/Q728944","display_name":"Inclusion (mineral)","level":2,"score":0.4375263452529907},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.39424604177474976},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.36403143405914307},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34059035778045654},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.22908064723014832},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10036230087280273},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09556907415390015},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08429700136184692},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3091478.3098871","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3091478.3098871","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2017 ACM on Web Science Conference","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7799999713897705,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W2038507439","https://openalex.org/W2159205954","https://openalex.org/W2563852449","https://openalex.org/W2950037719","https://openalex.org/W2951737564"],"related_works":["https://openalex.org/W3008339103","https://openalex.org/W2404647514","https://openalex.org/W1667647204","https://openalex.org/W4247536566","https://openalex.org/W3119814709","https://openalex.org/W2018477250","https://openalex.org/W1508895727","https://openalex.org/W4241418540","https://openalex.org/W2725786787","https://openalex.org/W4283160672"],"abstract_inverted_index":{"Wagstaff":[0],"(2012)":[1],"draws":[2],"attention":[3,100],"to":[4,23,74,104],"the":[5,34,45,54,57,75,85,88,91,105,111,125,128,131,148,167],"pervasiveness":[6],"of":[7,36,44,59,77,84,93,113,127,161,169],"abstract":[8,154],"evaluation":[9],"metrics":[10,20,115],"that":[11,51,116,144],"explicitly":[12],"ignore":[13],"or":[14],"remove":[15],"problem":[16,79],"specifics.":[17],"While":[18,90],"such":[19,156],"allow":[21],"practitioners":[22],"compare":[24],"numbers":[25],"across":[26],"application":[27],"domains,":[28],"they":[29],"offer":[30],"limited":[31],"insight":[32],"into":[33],"impact":[35],"algorithmic":[37],"decisions":[38],"on":[39,135,139,166],"humans":[40,121],"and":[41,130,171],"their":[42,66],"perception":[43,160],"algorithm's":[46],"correctness.":[47],"Even":[48],"for":[49,110],"problems":[50],"are":[52],"mathematically":[53],"same,":[55],"both":[56],"real-cost":[58,92],"(mathematically)":[60],"identical":[61],"errors,":[62],"as":[63,65,81,83,150,157],"well":[64,82],"perceived-cost":[67,106],"by":[68,152],"users,":[69],"may":[70],"significantly":[71],"vary":[72],"according":[73],"specifics":[76],"each":[78],"domain,":[80],"user":[86,132,159,172],"perceiving":[87],"result.":[89],"errors":[94,170],"has":[95,101],"been":[96,102],"considered":[97],"previously,":[98],"little":[99],"paid":[103],"issue.":[107],"We":[108],"advocate":[109],"inclusion":[112],"human-centered":[114],"elicit":[117],"error":[118],"costs":[119],"from":[120,122],"two":[123],"perspectives:":[124],"nature":[126,168],"error,":[129],"context.":[133],"Focusing":[134],"hate":[136],"speech":[137],"detection":[138],"social":[140],"media,":[141],"we":[142],"demonstrate":[143],"even":[145],"when":[146],"fixing":[147],"performance":[149],"measured":[151],"an":[153],"metric":[155],"precision,":[158],"correctness":[162],"varies":[163],"greatly":[164],"depending":[165],"characteristics.":[173]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":1}],"updated_date":"2026-03-09T08:58:05.943551","created_date":"2025-10-10T00:00:00"}