{"id":"https://openalex.org/W7162769724","doi":"https://doi.org/10.1016/j.ins.2026.123697","title":"Enhancing classifier evaluation: A difficulty-aware benchmarking strategy based on ability and robustness","display_name":"Enhancing classifier evaluation: A difficulty-aware benchmarking strategy based on ability and robustness","publication_year":2026,"publication_date":"2026-05-29","ids":{"openalex":"https://openalex.org/W7162769724","doi":"https://doi.org/10.1016/j.ins.2026.123697"},"language":"en","primary_location":{"id":"doi:10.1016/j.ins.2026.123697","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ins.2026.123697","pdf_url":null,"source":{"id":"https://openalex.org/S192650101","display_name":"Information Sciences","issn_l":"0020-0255","issn":["0020-0255","1872-6291"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Sciences","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1016/j.ins.2026.123697","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137377160","display_name":"Lucas Felipe Ferraro Cardoso","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161302","display_name":"Vale Technological Institute","ror":"https://ror.org/05wnasr61","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210161302"]},{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":true,"raw_author_name":"Lucas Felipe Ferraro Cardoso","raw_affiliation_strings":["Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-3838-3214","affiliations":[{"raw_affiliation_string":"Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","institution_ids":["https://openalex.org/I59606676"]},{"raw_affiliation_string":"Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil","institution_ids":["https://openalex.org/I4210161302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103073179","display_name":"Vitor Cirilo Araujo Santos","orcid":"https://orcid.org/0000-0002-7960-3079"},"institutions":[{"id":"https://openalex.org/I4210161302","display_name":"Vale Technological Institute","ror":"https://ror.org/05wnasr61","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210161302"]},{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Vitor Cirilo Araujo Santos","raw_affiliation_strings":["Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-7960-3079","affiliations":[{"raw_affiliation_string":"Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","institution_ids":["https://openalex.org/I59606676"]},{"raw_affiliation_string":"Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil","institution_ids":["https://openalex.org/I4210161302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024245764","display_name":"Jos\u00e9 de Sousa Ribeiro Filho","orcid":"https://orcid.org/0000-0002-8836-4188"},"institutions":[{"id":"https://openalex.org/I4210112416","display_name":"Instituto Federal de Educa\u00e7\u00e3o, Ci\u00eancia e Tecnologia do Par\u00e1","ror":"https://ror.org/02239nd21","country_code":"BR","type":"education","lineage":["https://openalex.org/I1293487690","https://openalex.org/I2801200668","https://openalex.org/I4210112416"]},{"id":"https://openalex.org/I4210161302","display_name":"Vale Technological Institute","ror":"https://ror.org/05wnasr61","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210161302"]},{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Jos\u00e9 de Sousa Ribeiro Filho","raw_affiliation_strings":["Federal Institute of Par\u00e1 - IFPA, Bel\u00e9m, 67125-000, PA, Brazil","Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0002-8836-4188","affiliations":[{"raw_affiliation_string":"Federal Institute of Par\u00e1 - IFPA, Bel\u00e9m, 67125-000, PA, Brazil","institution_ids":["https://openalex.org/I4210112416"]},{"raw_affiliation_string":"Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","institution_ids":["https://openalex.org/I59606676"]},{"raw_affiliation_string":"Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil","institution_ids":["https://openalex.org/I4210161302"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068068844","display_name":"Regiane S. Kawasaki Franc\u00eas","orcid":"https://orcid.org/0000-0003-3958-064X"},"institutions":[{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Regiane Silva Kawasaki Franc\u00eas","raw_affiliation_strings":["Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-3958-064X","affiliations":[{"raw_affiliation_string":"Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","institution_ids":["https://openalex.org/I59606676"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083998049","display_name":"Ricardo B. C. Prud\u00eancio","orcid":"https://orcid.org/0000-0001-7084-1233"},"institutions":[{"id":"https://openalex.org/I25112270","display_name":"Universidade Federal de Pernambuco","ror":"https://ror.org/047908t24","country_code":"BR","type":"education","lineage":["https://openalex.org/I25112270"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Ricardo Bastos Cavalcante Prud\u00eancio","raw_affiliation_strings":["Federal University of Pernambuco - UFPE, Recife, 50670-901, PE, Brazil"],"raw_orcid":"https://orcid.org/0000-0001-7084-1233","affiliations":[{"raw_affiliation_string":"Federal University of Pernambuco - UFPE, Recife, 50670-901, PE, Brazil","institution_ids":["https://openalex.org/I25112270"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5137366473","display_name":"Ronnie Cley de Oliveira Alves","orcid":null},"institutions":[{"id":"https://openalex.org/I4210161302","display_name":"Vale Technological Institute","ror":"https://ror.org/05wnasr61","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210161302"]},{"id":"https://openalex.org/I59606676","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818","country_code":"BR","type":"education","lineage":["https://openalex.org/I59606676"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Ronnie Cley de Oliveira Alves","raw_affiliation_strings":["Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-4139-0562","affiliations":[{"raw_affiliation_string":"Federal University of Par\u00e1 - UFPA, Bel\u00e9m, 66075-11, PA, Brazil","institution_ids":["https://openalex.org/I59606676"]},{"raw_affiliation_string":"Vale Technological Institute - ITV, Bel\u00e9m, 66055-09, PA, Brazil","institution_ids":["https://openalex.org/I4210161302"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5137377160"],"corresponding_institution_ids":["https://openalex.org/I4210161302","https://openalex.org/I59606676"],"apc_list":{"value":3330,"currency":"USD","value_usd":3330},"apc_paid":{"value":3330,"currency":"USD","value_usd":3330},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.66847463,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"754","issue":null,"first_page":"123697","last_page":"123697"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.3716000020503998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.3716000020503998,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11652","display_name":"Imbalanced Data Classification Techniques","score":0.13009999692440033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11674","display_name":"Sports Analytics and Performance","score":0.09279999881982803,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.897599995136261},{"id":"https://openalex.org/keywords/classifier","display_name":"Classifier (UML)","score":0.6363999843597412},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6159999966621399},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5942999720573425},{"id":"https://openalex.org/keywords/random-forest","display_name":"Random forest","score":0.49630001187324524}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.897599995136261},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7918000221252441},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.697700023651123},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.6363999843597412},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6159999966621399},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6033999919891357},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5942999720573425},{"id":"https://openalex.org/C169258074","wikidata":"https://www.wikidata.org/wiki/Q245748","display_name":"Random forest","level":2,"score":0.49630001187324524},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.46799999475479126},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.3662000000476837},{"id":"https://openalex.org/C12267149","wikidata":"https://www.wikidata.org/wiki/Q282453","display_name":"Support vector machine","level":2,"score":0.3009999990463257},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.2630000114440918}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1016/j.ins.2026.123697","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ins.2026.123697","pdf_url":null,"source":{"id":"https://openalex.org/S192650101","display_name":"Information Sciences","issn_l":"0020-0255","issn":["0020-0255","1872-6291"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Sciences","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1016/j.ins.2026.123697","is_oa":true,"landing_page_url":"https://doi.org/10.1016/j.ins.2026.123697","pdf_url":null,"source":{"id":"https://openalex.org/S192650101","display_name":"Information Sciences","issn_l":"0020-0255","issn":["0020-0255","1872-6291"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320990","host_organization_name":"Elsevier BV","host_organization_lineage":["https://openalex.org/P4310320990"],"host_organization_lineage_names":["Elsevier BV"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information Sciences","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320316037","display_name":"Instituto Tecnol\u00f3gico Vale","ror":"https://ror.org/05wnasr61"},{"id":"https://openalex.org/F4320324376","display_name":"Universidade Federal do Par\u00e1","ror":"https://ror.org/03q9sr818"},{"id":"https://openalex.org/F4320324901","display_name":"Universidade Federal de Pernambuco","ror":"https://ror.org/047908t24"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W1680939745","https://openalex.org/W1992065666","https://openalex.org/W2050627483","https://openalex.org/W2055089682","https://openalex.org/W2116825089","https://openalex.org/W2125693710","https://openalex.org/W2132862423","https://openalex.org/W2150468603","https://openalex.org/W2161336914","https://openalex.org/W2912981918","https://openalex.org/W3102606356","https://openalex.org/W4365445794","https://openalex.org/W4389966230","https://openalex.org/W4396712983","https://openalex.org/W4407975052","https://openalex.org/W4412002212","https://openalex.org/W4412751168","https://openalex.org/W4413371190","https://openalex.org/W4416133463","https://openalex.org/W7134285814"],"related_works":[],"abstract_inverted_index":{"Benchmarking":[0],"is":[1],"a":[2,22,65,117,145],"fundamental":[3],"practice":[4],"in":[5,86],"machine":[6],"learning":[7],"(ML)":[8],"for":[9],"comparing":[10],"the":[11,25,76,130,138,151,159,165,172],"performance":[12,95,102],"of":[13,28,123,137,150,174],"classification":[14],"algorithms.":[15],"However,":[16],"traditional":[17],"evaluation":[18,67,156,185],"methods":[19],"often":[20],"overlook":[21],"critical":[23],"aspect:":[24],"joint":[26],"consideration":[27],"dataset":[29,181],"complexity":[30],"and":[31,107,120,143,183,191],"an":[32],"algorithm\u2019s":[33],"ability":[34,92,167],"to":[35,53,82],"generalize.":[36],"Without":[37],"this":[38,60,62],"dual":[39],"perspective,":[40],"assessments":[41],"may":[42],"favor":[43],"models":[44],"that":[45,69,134,144,187],"perform":[46],"well":[47],"on":[48,94,180],"easy":[49],"instances":[50],"while":[51,99],"failing":[52],"capture":[54],"their":[55],"true":[56],"robustness.":[57],"To":[58],"address":[59],"limitation,":[61],"study":[63,128],"introduces":[64],"novel":[66],"methodology":[68],"combines":[70],"Item":[71],"Response":[72],"Theory":[73],"(IRT)":[74],"with":[75,148],"Glicko-2":[77,100],"rating":[78],"system,":[79],"originally":[80],"developed":[81],"measure":[83,122],"player":[84],"strength":[85],"competitive":[87],"games.":[88],"IRT":[89],"assesses":[90],"classifier":[91,192],"based":[93],"over":[96],"difficult":[97],"instances,":[98],"updates":[101],"metrics\u2014such":[103],"as":[104],"rating,":[105],"deviation,":[106],"volatility\u2014via":[108],"simulated":[109],"tournaments":[110],"between":[111],"classifiers.":[112],"This":[113],"combined":[114],"approach":[115],"provides":[116],"more":[118],"robust":[119],"difficulty-aware":[121],"algorithm":[124],"capability.":[125],"A":[126],"case":[127],"using":[129],"OpenML-CC18":[131],"benchmark":[132,176],"showed":[133],"only":[135],"16.66%":[136],"datasets":[139,153],"are":[140],"truly":[141],"challenging":[142],"reduced":[146],"subset":[147],"50%":[149],"original":[152],"offers":[154],"comparable":[155],"power.":[157],"Among":[158],"algorithms":[160],"tested,":[161],"Random":[162],"Forest":[163],"achieved":[164],"highest":[166],"score.":[168],"The":[169],"results":[170],"highlight":[171],"importance":[173],"improving":[175],"design":[177],"by":[178],"focusing":[179],"quality":[182],"adopting":[184],"strategies":[186],"reflect":[188],"both":[189],"difficulty":[190],"proficiency.":[193]},"counts_by_year":[],"updated_date":"2026-06-03T06:16:58.514037","created_date":"2026-05-30T00:00:00"}