{"id":"https://openalex.org/W7155069402","doi":"https://doi.org/10.1109/access.2026.3686088","title":"Beyond the Leaderboard: A Survey of the Science of Evaluation, Benchmarking, and Methodologies for Large Language Models","display_name":"Beyond the Leaderboard: A Survey of the Science of Evaluation, Benchmarking, and Methodologies for Large Language Models","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7155069402","doi":"https://doi.org/10.1109/access.2026.3686088"},"language":"en","primary_location":{"id":"doi:10.1109/access.2026.3686088","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3686088","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2026.3686088","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057569676","display_name":"Saeid Sheikhi","orcid":"https://orcid.org/0000-0002-3600-966X"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Saeid Sheikhi","raw_affiliation_strings":["University of Oulu, Oulu, Finland"],"raw_orcid":"https://orcid.org/0000-0002-3600-966X","affiliations":[{"raw_affiliation_string":"University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051101152","display_name":"Lauri Lov\u00e9n","orcid":"https://orcid.org/0000-0001-9475-4839"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Lauri Lov\u00e9n","raw_affiliation_strings":["University of Oulu, Oulu, Finland"],"raw_orcid":"https://orcid.org/0000-0001-9475-4839","affiliations":[{"raw_affiliation_string":"University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051336752","display_name":"Panos Kostakos","orcid":"https://orcid.org/0000-0002-8545-599X"},"institutions":[{"id":"https://openalex.org/I98381234","display_name":"University of Oulu","ror":"https://ror.org/03yj89h83","country_code":"FI","type":"education","lineage":["https://openalex.org/I98381234"]}],"countries":["FI"],"is_corresponding":false,"raw_author_name":"Panos Kostakos","raw_affiliation_strings":["University of Oulu, Oulu, Finland"],"raw_orcid":"https://orcid.org/0000-0002-8545-599X","affiliations":[{"raw_affiliation_string":"University of Oulu, Oulu, Finland","institution_ids":["https://openalex.org/I98381234"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98381234"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.54092219,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"14","issue":null,"first_page":"66493","last_page":"66515"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.1915999948978424,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.1915999948978424,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.06729999929666519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.03849999979138374,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.32919999957084656},{"id":"https://openalex.org/keywords/data-modeling","display_name":"Data modeling","score":0.3260999917984009},{"id":"https://openalex.org/keywords/context-model","display_name":"Context model","score":0.28700000047683716},{"id":"https://openalex.org/keywords/modeling-language","display_name":"Modeling language","score":0.274399995803833},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.2705000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7214000225067139},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3677999973297119},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.32919999957084656},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.3260999917984009},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3095000088214874},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2939999997615814},{"id":"https://openalex.org/C183322885","wikidata":"https://www.wikidata.org/wiki/Q17007702","display_name":"Context model","level":3,"score":0.28700000047683716},{"id":"https://openalex.org/C179603123","wikidata":"https://www.wikidata.org/wiki/Q1941921","display_name":"Modeling language","level":3,"score":0.274399995803833},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2705000042915344},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C2983448237","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Language understanding","level":2,"score":0.2549999952316284},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.25380000472068787}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2026.3686088","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3686088","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:e52819840786414493a9610f25271cf5","is_oa":true,"landing_page_url":"https://doaj.org/article/e52819840786414493a9610f25271cf5","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 14, Pp 66493-66515 (2026)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2026.3686088","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2026.3686088","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.7215011715888977,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G1538652318","display_name":null,"funder_award_id":"A91867","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G3187350251","display_name":"6G Bridge \u2014 Oulun Yliopisto (8754/31/2022)","funder_award_id":"8754/31/2022","funder_id":"https://openalex.org/F4320328501","funder_display_name":"Business Finland"},{"id":"https://openalex.org/G4230935547","display_name":null,"funder_award_id":"318927","funder_id":"https://openalex.org/F4320321108","funder_display_name":"Academy of Finland"},{"id":"https://openalex.org/G5353076939","display_name":null,"funder_award_id":"A81568","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"}],"funders":[{"id":"https://openalex.org/F4320321108","display_name":"Academy of Finland","ror":"https://ror.org/05k73zm37"},{"id":"https://openalex.org/F4320328501","display_name":"Business Finland","ror":"https://ror.org/05bgf9v38"},{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"field":[1],"of":[2,14,20,57,114,128,142,156,164,195,209,215,248],"artificial":[3,278],"intelligence":[4],"is":[5,33,50,105],"grappling":[6],"with":[7,201],"an":[8],"evaluation":[9,78,130,165,187,249,253],"crisis.":[10],"Despite":[11],"the":[12,17,92,110,126,152,196,199,206,212,216,220,236,241,263],"proliferation":[13],"benchmarks":[15,98],"and":[16,60,99,176,180,185,219,226,246,265,271],"relentless":[18],"pursuit":[19],"state-of-the-art":[21],"scores":[22],"on":[23,43,103,205],"public":[24],"leaderboards,":[25],"confidence":[26],"in":[27,277],"what":[28],"these":[29],"metrics":[30,150],"truly":[31],"measure":[32],"eroding.":[34],"Large":[35],"Language":[36],"Models":[37],"(LLMs)":[38],"now":[39],"consistently":[40],"outperform":[41],"humans":[42],"established":[44],"benchmarks.":[45],"However,":[46],"their":[47],"real-world":[48],"utility":[49],"often":[51],"undermined":[52],"by":[53,96,131],"brittleness,":[54],"a":[55,80,85,100,122,139,161,192,202,232,255,267],"lack":[56],"common":[58],"sense,":[59],"susceptibility":[61],"to":[62,108,120,151,239,260],"generating":[63],"harmful":[64],"or":[65],"factually":[66],"incorrect":[67],"content.":[68],"This":[69,117],"paper":[70],"argues":[71],"that":[72,91],"this":[73],"disconnect":[74],"stems":[75],"from":[76,147],"treating":[77],"as":[79,254],"competitive":[81],"afterthought":[82],"rather":[83],"than":[84],"rigorous":[86],"metrological":[87],"science.":[88],"We":[89],"contend":[90],"current":[93],"paradigm,":[94],"characterized":[95],"static":[97,173],"narrow":[101],"focus":[102,204],"accuracy,":[104],"fundamentally":[106],"ill-equipped":[107],"assess":[109],"complex,":[111],"multifaceted":[112],"capabilities":[113],"modern":[115],"LLMs.":[116],"survey":[118],"aims":[119],"provide":[121],"comprehensive":[123],"reference":[124],"for":[125,274],"science":[127],"LLM":[129],"making":[132],"four":[133],"principal":[134],"contributions.":[135],"First,":[136],"we":[137,159,190,230,258],"offer":[138],"historical":[140],"analysis":[141,194],"evaluation,":[143,178,182,184],"tracing":[144],"its":[145],"evolution":[146],"traditional":[148],"lexical-overlap":[149],"massive,":[153],"holistic":[154],"frameworks":[155],"today.":[157],"Second,":[158],"introduce":[160],"systematic":[162],"taxonomy":[163],"methodologies,":[166],"classifying":[167],"them":[168],"into":[169],"five":[170],"key":[171],"paradigms:":[172],"benchmarks,":[174],"dynamic":[175],"adaptive":[177],"interactive":[179],"agentic":[181],"human-in-the-loop":[183],"model-based":[186],"(LLM-as-a-Judge).":[188],"Third,":[189],"conduct":[191],"critical":[193],"pathologies":[197],"plaguing":[198],"field,":[200],"deep":[203],"existential":[207],"threat":[208],"data":[210],"contamination,":[211],"perverse":[213],"incentives":[214],"leaderboard":[217,264],"obsession,":[218],"pervasive":[221],"mismatch":[222],"between":[223],"benchmark":[224],"performance":[225],"true":[227],"capability.":[228],"Finally,":[229],"propose":[231],"new":[233],"conceptual":[234],"framework,":[235],"Benchmark":[237],"Lifecycle,":[238],"professionalize":[240],"design,":[242],"validation,":[243],"deployment,":[244],"monitoring,":[245],"retirement":[247],"instruments.":[250],"By":[251],"framing":[252],"scientific":[256],"discipline,":[257],"aim":[259],"move":[261],"beyond":[262],"build":[266],"more":[268],"robust,":[269],"reliable,":[270],"trustworthy":[272],"foundation":[273],"measuring":[275],"progress":[276],"intelligence.":[279]},"counts_by_year":[],"updated_date":"2026-07-13T07:31:44.756512","created_date":"2026-04-22T00:00:00"}