{"id":"https://openalex.org/W4408565761","doi":"https://doi.org/10.1109/icdmw65004.2024.00115","title":"Beyond Overall Accuracy: Exploring Per-Class Performance Reveals Robustness and Safety Gaps in Deep Models","display_name":"Beyond Overall Accuracy: Exploring Per-Class Performance Reveals Robustness and Safety Gaps in Deep Models","publication_year":2024,"publication_date":"2024-12-09","ids":{"openalex":"https://openalex.org/W4408565761","doi":"https://doi.org/10.1109/icdmw65004.2024.00115"},"language":"en","primary_location":{"id":"doi:10.1109/icdmw65004.2024.00115","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw65004.2024.00115","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Data Mining Workshops (ICDMW)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058628846","display_name":"Kamil Szyc","orcid":"https://orcid.org/0000-0001-6723-271X"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":true,"raw_author_name":"Kamil Szyc","raw_affiliation_strings":["Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland"],"affiliations":[{"raw_affiliation_string":"Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010635837","display_name":"Tomasz Walkowiak","orcid":null},"institutions":[{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]},{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Tomasz Walkowiak","raw_affiliation_strings":["Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland"],"affiliations":[{"raw_affiliation_string":"Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5047219170","display_name":"Henryk Maciejewski","orcid":"https://orcid.org/0000-0002-8405-9987"},"institutions":[{"id":"https://openalex.org/I11923345","display_name":"Wroc\u0142aw University of Science and Technology","ror":"https://ror.org/008fyn775","country_code":"PL","type":"education","lineage":["https://openalex.org/I11923345"]},{"id":"https://openalex.org/I686019","display_name":"AGH University of Krakow","ror":"https://ror.org/00bas1c41","country_code":"PL","type":"education","lineage":["https://openalex.org/I686019"]}],"countries":["PL"],"is_corresponding":false,"raw_author_name":"Henryk Maciejewski","raw_affiliation_strings":["Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland"],"affiliations":[{"raw_affiliation_string":"Wroclaw University of Science and Technology,Faculty of Information and Communication Technology,Wrocław,Poland","institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5058628846"],"corresponding_institution_ids":["https://openalex.org/I11923345","https://openalex.org/I686019"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.262838,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"833","last_page":"841"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9764999747276306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.9764999747276306,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11512","display_name":"Anomaly Detection Techniques and Applications","score":0.9422000050544739,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9362000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7950342297554016},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5923338532447815},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4212776720523834},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3573053479194641},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.32577526569366455},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.20112663507461548}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7950342297554016},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5923338532447815},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4212776720523834},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3573053479194641},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.32577526569366455},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.20112663507461548},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icdmw65004.2024.00115","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icdmw65004.2024.00115","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Data Mining Workshops (ICDMW)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Climate action","id":"https://metadata.un.org/sdg/13","score":0.4399999976158142}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":30,"referenced_works":["https://openalex.org/W1493526108","https://openalex.org/W1978394996","https://openalex.org/W2047643928","https://openalex.org/W2081580037","https://openalex.org/W2117539524","https://openalex.org/W2194775991","https://openalex.org/W2787894218","https://openalex.org/W2963163009","https://openalex.org/W3037492894","https://openalex.org/W3045492832","https://openalex.org/W3177096435","https://openalex.org/W3199086051","https://openalex.org/W3202586005","https://openalex.org/W4312443924","https://openalex.org/W6748304040","https://openalex.org/W6757555829","https://openalex.org/W6762718338","https://openalex.org/W6764990469","https://openalex.org/W6766795243","https://openalex.org/W6775180354","https://openalex.org/W6779989637","https://openalex.org/W6781574218","https://openalex.org/W6784147671","https://openalex.org/W6784333009","https://openalex.org/W6787380718","https://openalex.org/W6793164127","https://openalex.org/W6795475546","https://openalex.org/W6810207935","https://openalex.org/W6810300553","https://openalex.org/W6858547017"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Commonly":[0],"used":[1,20],"measures":[2,47,122,219],"of":[3,5,71,80,93,100,116,171,181,189,242],"performance":[4,34,79,218,226],"Deep":[6],"Neural":[7],"Networks":[8],"(DNNs)":[9],"are":[10,164],"averaged":[11,46],"over":[12],"all":[13],"cases/classes.":[14],"For":[15],"instance,":[16],"overall":[17,229],"accuracy":[18,105],"is":[19,183,205],"in":[21,67,139,162,207],"hyper-parameter":[22],"selection,":[23],"network":[24],"regularization":[25],"using":[26],"the":[27,32,36,54,68,72,76,94,128,136,168,179,187,215,228,235,239],"validation":[28],"dataset,":[29],"or":[30],"assessing":[31],"model\u2019s":[33,129],"on":[35,90,107,151],"test":[37,108],"set.":[38],"In":[39],"this":[40],"work,":[41],"we":[42,154],"demonstrate":[43],"that":[44,113,156],"these":[45,152,174,193],"hide":[48],"critical":[49],"model":[50,202],"flaws":[51],"related":[52],"to":[53,61,167,186],"worst":[55,77,224],"class":[56,78,225],"performance,":[57],"which":[58,204],"can":[59,123],"lead":[60],"severe":[62],"robustness":[63,117,130,160],"and":[64,118,131,159,210],"safety":[65,132,236],"gaps":[66],"real-life":[69],"deployment":[70,241],"models.":[73,244],"We":[74,110,134],"analyzed":[75],"several":[81],"State-of-the-Art":[82],"(SoTA)":[83],"models":[84,142],"(ResNet,":[85],"ConvNeXt,":[86],"EfficientNet,":[87],"ViT)":[88],"trained":[89,143],"ImageNet.":[91],"Regardless":[92],"representation-generating":[95],"model,":[96],"a":[97],"common":[98],"subset":[99],"classes":[101],"realize":[102],"disproportionately":[103],"low":[104,169],"(0.2-0.5)":[106],"data.":[109],"also":[111,222],"showed":[112],"per-class":[114],"analysis":[115,198],"out-of-distribution":[119],"(OOD)":[120],"generalization":[121,158],"bring":[124],"valuable":[125],"insights":[126],"into":[127],"issues.":[133],"find":[135],"same":[137],"effects":[138],"text":[140],"classification":[141],"with":[144,238],"different":[145],"representations":[146,172,182],"(e.g.,":[147],"Bert,":[148],"embedding-based).":[149],"Based":[150],"results,":[153],"conclude":[155],"(i)":[157],"issues":[161],"DNNs":[163],"primarily":[165,184],"due":[166,185],"quality":[170,180,188],"for":[173,192],"weak":[175],"classes,":[176,196],"(ii)":[177],"since":[178],"training":[190],"data":[191,209],"weak,":[194],"non-robust":[195],"our":[197],"enables":[199],"more":[200],"targeted":[201],"improvement":[203],"crucial":[206],"static":[208],"continual":[211],"learning":[212],"scenarios,":[213],"(iii)":[214],"reported":[216],"DNN":[217,243],"should":[220],"mandatory":[221],"include":[223],"as":[227],"accuracy/robustness/generalization":[230],"does":[231],"not":[232],"fairly":[233],"represent":[234],"risks":[237],"open-world":[240]},"counts_by_year":[],"updated_date":"2025-12-28T23:10:05.387466","created_date":"2025-10-10T00:00:00"}