{"id":"https://openalex.org/W4415227752","doi":"https://doi.org/10.1609/aies.v8i1.36595","title":"Can We Trust AI Benchmarks? An Interdisciplinary Review of Current Issues in AI Evaluation","display_name":"Can We Trust AI Benchmarks? An Interdisciplinary Review of Current Issues in AI Evaluation","publication_year":2025,"publication_date":"2025-10-15","ids":{"openalex":"https://openalex.org/W4415227752","doi":"https://doi.org/10.1609/aies.v8i1.36595"},"language":"en","primary_location":{"id":"doi:10.1609/aies.v8i1.36595","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36595","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36595/38733","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://ojs.aaai.org/index.php/AIES/article/download/36595/38733","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058592374","display_name":"Maria Eriksson","orcid":"https://orcid.org/0000-0002-7261-6643"},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Maria Eriksson","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035879496","display_name":"Erasmo Purificato","orcid":"https://orcid.org/0000-0002-5506-3020"},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Erasmo Purificato","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035515300","display_name":"Arman Noroozian","orcid":null},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Arman Noroozian","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075867525","display_name":"Jo\u00e3o Vinagre","orcid":"https://orcid.org/0000-0001-6219-3977"},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Jo\u00e3o Vinagre","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108354434","display_name":"Guillaume Chaslot","orcid":null},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Guillaume Chaslot","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103887030","display_name":"Emilia G\u00f3mez","orcid":null},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Emilia Gomez","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016004555","display_name":"David Fern\u00e1ndez Llorca","orcid":"https://orcid.org/0000-0003-2433-7110"},"institutions":[{"id":"https://openalex.org/I2800387288","display_name":"European Commission","ror":"https://ror.org/00k4n6c32","country_code":"BE","type":"government","lineage":["https://openalex.org/I1320481043","https://openalex.org/I2800387288"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"David Fernandez-Llorca","raw_affiliation_strings":["European Commission, Joint Research Centre"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"European Commission, Joint Research Centre","institution_ids":["https://openalex.org/I2800387288"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I2800387288"],"apc_list":null,"apc_paid":null,"fwci":25.0952,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.99716441,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"8","issue":"1","first_page":"850","last_page":"864"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.8723999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.8723999857902527,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9333999752998352},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6712999939918518},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.6220999956130981},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5946000218391418},{"id":"https://openalex.org/keywords/accountability","display_name":"Accountability","score":0.5263000130653381}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9333999752998352},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6712999939918518},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.6220999956130981},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5989999771118164},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5946000218391418},{"id":"https://openalex.org/C2776007630","wikidata":"https://www.wikidata.org/wiki/Q2798912","display_name":"Accountability","level":2,"score":0.5263000130653381},{"id":"https://openalex.org/C127627568","wikidata":"https://www.wikidata.org/wiki/Q1639361","display_name":"Sociotechnical system","level":2,"score":0.4700999855995178},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.4575999975204468},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4153999984264374},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.41029998660087585},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3792000114917755},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3781999945640564},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.3402000069618225},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.31290000677108765},{"id":"https://openalex.org/C157170001","wikidata":"https://www.wikidata.org/wiki/Q4781507","display_name":"Applications of artificial intelligence","level":2,"score":0.29330000281333923},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.29249998927116394}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1609/aies.v8i1.36595","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36595","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36595/38733","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},{"id":"pmh:oai:ojs.aaai.org:article/36595","is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AIES/article/view/36595","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"3065-8365","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1609/aies.v8i1.36595","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aies.v8i1.36595","pdf_url":"https://ojs.aaai.org/index.php/AIES/article/download/36595/38733","source":{"id":"https://openalex.org/S5407048695","display_name":"Proceedings of the AAAI/ACM Conference on AI Ethics and Society","issn_l":"3065-8365","issn":["3065-8365"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415227752.pdf","grobid_xml":"https://content.openalex.org/works/W4415227752.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Quantitative":[0],"Artificial":[1],"Intelligence":[2],"(AI)":[3],"Benchmarks":[4],"have":[5],"emerged":[6],"as":[7,61,107,128,174],"fundamental":[8],"tools":[9],"for":[10,145],"evaluating":[11,132],"the":[12,25,89,100,185,211,242,251],"performance,":[13],"capability,":[14],"and":[15,20,30,51,67,102,116,152,156,182,202,235,244],"safety":[16,66],"of":[17,27,76,104,166,187,213,221,246,253],"AI":[18,28,134,147,248],"models":[19,135,148],"systems.":[21],"Currently,":[22],"they":[23,55],"shape":[24],"direction":[26],"development":[29],"are":[31,149,196],"playing":[32],"an":[33,73,129,219],"increasingly":[34,150],"prominent":[35],"role":[36],"in":[37,83,88,99,109,169,233],"regulatory":[38],"frameworks.":[39],"As":[40],"their":[41],"influence":[42],"grows,":[43],"however,":[44],"so":[45],"too":[46],"does":[47],"concerns":[48],"about":[49,77],"how":[50,146,193],"with":[52,123,154,184,224],"what":[53],"effects":[54],"evaluate":[56],"highly":[57],"sensitive":[58],"topics":[59],"such":[60,173],"capabilities,":[62,65],"including":[63],"high-impact":[64],"systemic":[68,167],"risks.":[69],"This":[70],"paper":[71],"presents":[72],"interdisciplinary":[74],"meta-review":[75],"110":[78],"studies":[79],"that":[80,141,205],"discuss":[81],"shortcomings":[82],"quantitative":[84,247],"benchmarking":[85,171,226],"practices,":[86,172],"published":[87],"last":[90],"10":[91],"years.":[92],"It":[93],"brings":[94],"together":[95],"many":[96],"fine-grained":[97],"issues":[98,126],"design":[101],"application":[103],"benchmarks":[105,234,249],"(such":[106,127],"biases":[108],"dataset":[110],"creation,":[111],"inadequate":[112],"documentation,":[113],"data":[114],"contamination,":[115],"failures":[117],"to":[118,137,143,237,240],"distinguish":[119],"signal":[120],"from":[121],"noise)":[122],"broader":[124,214],"sociotechnical":[125],"over-focus":[130],"on":[131],"text-based":[133],"according":[136],"one-time":[138],"testing":[139],"logic":[140],"fails":[142],"account":[144],"multimodal":[151],"interact":[153],"humans":[155],"other":[157],"technical":[158],"systems).":[159],"Our":[160],"review":[161],"also":[162],"highlights":[163],"a":[164],"series":[165],"flaws":[168],"current":[170],"misaligned":[175],"incentives,":[176],"construct":[177],"validity":[178],"issues,":[179],"unknown":[180],"unknowns,":[181],"problems":[183],"gaming":[186],"benchmark":[188,194],"results.":[189],"Furthermore,":[190],"it":[191],"underscores":[192],"practices":[195],"fundamentally":[197],"shaped":[198],"by":[199],"cultural,":[200],"commercial":[201],"competitive":[203],"dynamics":[204],"often":[206],"prioritise":[207],"state-of-the-art":[208],"performance":[209],"at":[210],"expense":[212],"societal":[215],"concerns.":[216],"By":[217],"providing":[218],"overview":[220],"risks":[222],"associated":[223],"existing":[225],"procedures,":[227],"we":[228],"problematise":[229],"disproportionate":[230],"trust":[231],"placed":[232],"contribute":[236],"ongoing":[238],"efforts":[239],"improve":[241],"accountability":[243],"relevance":[245],"within":[250],"complexities":[252],"real-world":[254],"scenarios.":[255]},"counts_by_year":[{"year":2026,"cited_by_count":11},{"year":2025,"cited_by_count":4}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-16T00:00:00"}