{"id":"https://openalex.org/W4406458640","doi":"https://doi.org/10.1109/bigdata62323.2024.10825051","title":"Dynamic Intelligence Assessment: Benchmarking LLMs on the Road to AGI with a Focus on Model Confidence","display_name":"Dynamic Intelligence Assessment: Benchmarking LLMs on the Road to AGI with a Focus on Model Confidence","publication_year":2024,"publication_date":"2024-12-15","ids":{"openalex":"https://openalex.org/W4406458640","doi":"https://doi.org/10.1109/bigdata62323.2024.10825051"},"language":"en","primary_location":{"id":"doi:10.1109/bigdata62323.2024.10825051","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825051","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073042391","display_name":"Norbert Tihanyi","orcid":"https://orcid.org/0000-0002-9002-5935"},"institutions":[{"id":"https://openalex.org/I4210087059","display_name":"Technology Innovation Institute","ror":"https://ror.org/001kv2y39","country_code":"AE","type":"facility","lineage":["https://openalex.org/I4210087059"]}],"countries":["AE"],"is_corresponding":true,"raw_author_name":"Norbert Tihanyi","raw_affiliation_strings":["Technology Innovation Institute,Abu Dhabi,United Arab Emirates"],"affiliations":[{"raw_affiliation_string":"Technology Innovation Institute,Abu Dhabi,United Arab Emirates","institution_ids":["https://openalex.org/I4210087059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052473439","display_name":"Tam\u00e1s Bisztray","orcid":"https://orcid.org/0000-0003-2626-3434"},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Tamas Bisztray","raw_affiliation_strings":["University of Oslo,Oslo,Norway"],"affiliations":[{"raw_affiliation_string":"University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114601640","display_name":"Richard A. Dubniczky","orcid":"https://orcid.org/0009-0003-3951-1932"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Richard A. Dubniczky","raw_affiliation_strings":["Eötvös Loránd University,Budapest,Hungary"],"affiliations":[{"raw_affiliation_string":"Eötvös Loránd University,Budapest,Hungary","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101342735","display_name":"Rebeka T\u00f3th","orcid":"https://orcid.org/0009-0000-9574-1896"},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Rebeka Toth","raw_affiliation_strings":["University of Oslo,Oslo,Norway"],"affiliations":[{"raw_affiliation_string":"University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069161234","display_name":"Bertalan Borsos","orcid":"https://orcid.org/0009-0000-8718-8285"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bertalan Borsos","raw_affiliation_strings":["Eötvös Loránd University,Budapest,Hungary"],"affiliations":[{"raw_affiliation_string":"Eötvös Loránd University,Budapest,Hungary","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5098768016","display_name":"Bilel Cherif","orcid":null},"institutions":[{"id":"https://openalex.org/I4210087059","display_name":"Technology Innovation Institute","ror":"https://ror.org/001kv2y39","country_code":"AE","type":"facility","lineage":["https://openalex.org/I4210087059"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Bilel Cherif","raw_affiliation_strings":["Technology Innovation Institute,Abu Dhabi,United Arab Emirates"],"affiliations":[{"raw_affiliation_string":"Technology Innovation Institute,Abu Dhabi,United Arab Emirates","institution_ids":["https://openalex.org/I4210087059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102012126","display_name":"Ridhi Jain","orcid":"https://orcid.org/0000-0002-6102-7114"},"institutions":[{"id":"https://openalex.org/I4210087059","display_name":"Technology Innovation Institute","ror":"https://ror.org/001kv2y39","country_code":"AE","type":"facility","lineage":["https://openalex.org/I4210087059"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Ridhi Jain","raw_affiliation_strings":["Technology Innovation Institute,Abu Dhabi,United Arab Emirates"],"affiliations":[{"raw_affiliation_string":"Technology Innovation Institute,Abu Dhabi,United Arab Emirates","institution_ids":["https://openalex.org/I4210087059"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114601641","display_name":"Lajos Muzsai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lajos Muzsai","raw_affiliation_strings":["Eötvös Loránd University,Budapest,Hungary"],"affiliations":[{"raw_affiliation_string":"Eötvös Loránd University,Budapest,Hungary","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026903935","display_name":"Mohamed Amine Ferrag","orcid":"https://orcid.org/0000-0002-0632-3172"},"institutions":[{"id":"https://openalex.org/I4210097536","display_name":"University of Guelma","ror":"https://ror.org/00xe6p546","country_code":"DZ","type":"education","lineage":["https://openalex.org/I4210097536"]}],"countries":["DZ"],"is_corresponding":false,"raw_author_name":"Mohamed Amine Ferrag","raw_affiliation_strings":["University of Guelma,Guelma,Algeria"],"affiliations":[{"raw_affiliation_string":"University of Guelma,Guelma,Algeria","institution_ids":["https://openalex.org/I4210097536"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114601639","display_name":"Ryan Marinelli","orcid":null},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Ryan Marinelli","raw_affiliation_strings":["University of Oslo,Oslo,Norway"],"affiliations":[{"raw_affiliation_string":"University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057689302","display_name":"Lucas C. Cordeiro","orcid":"https://orcid.org/0000-0002-6235-4272"},"institutions":[{"id":"https://openalex.org/I28407311","display_name":"University of Manchester","ror":"https://ror.org/027m9bs27","country_code":"GB","type":"education","lineage":["https://openalex.org/I28407311"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Lucas C. Cordeiro","raw_affiliation_strings":["The University of Manchester,Manchester,United Kingdom"],"affiliations":[{"raw_affiliation_string":"The University of Manchester,Manchester,United Kingdom","institution_ids":["https://openalex.org/I28407311"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056145687","display_name":"M\u00e9rouane Debbah","orcid":"https://orcid.org/0000-0001-8941-8080"},"institutions":[{"id":"https://openalex.org/I176601375","display_name":"Khalifa University of Science and Technology","ror":"https://ror.org/05hffr360","country_code":"AE","type":"education","lineage":["https://openalex.org/I176601375"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Merouane Debbah","raw_affiliation_strings":["Khalifa University,Abu Dhabi,United Arab Emirates"],"affiliations":[{"raw_affiliation_string":"Khalifa University,Abu Dhabi,United Arab Emirates","institution_ids":["https://openalex.org/I176601375"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051922069","display_name":"Vasileios Mavroeidis","orcid":"https://orcid.org/0000-0002-8188-3575"},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Vasileios Mavroeidis","raw_affiliation_strings":["University of Oslo,Oslo,Norway"],"affiliations":[{"raw_affiliation_string":"University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I184942183"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024678267","display_name":"Audun J\u00f8sang","orcid":"https://orcid.org/0000-0001-6337-2264"},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Audun J\u00f8sang","raw_affiliation_strings":["University of Oslo,Oslo,Norway"],"affiliations":[{"raw_affiliation_string":"University of Oslo,Oslo,Norway","institution_ids":["https://openalex.org/I184942183"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":14,"corresponding_author_ids":["https://openalex.org/A5073042391"],"corresponding_institution_ids":["https://openalex.org/I4210087059"],"apc_list":null,"apc_paid":null,"fwci":2.3173,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.90180555,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"3313","last_page":"3321"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9830999970436096,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.911899983882904,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8146910667419434},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.6607692241668701},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5481449365615845},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.1255321204662323},{"id":"https://openalex.org/keywords/marketing","display_name":"Marketing","score":0.10533159971237183},{"id":"https://openalex.org/keywords/optics","display_name":"Optics","score":0.07546654343605042}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8146910667419434},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.6607692241668701},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5481449365615845},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.1255321204662323},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.10533159971237183},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.07546654343605042},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/bigdata62323.2024.10825051","is_oa":false,"landing_page_url":"https://doi.org/10.1109/bigdata62323.2024.10825051","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2024 IEEE International Conference on Big Data (BigData)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":67,"referenced_works":["https://openalex.org/W1498436455","https://openalex.org/W2101105183","https://openalex.org/W2154652894","https://openalex.org/W2612690371","https://openalex.org/W2889787757","https://openalex.org/W2896457183","https://openalex.org/W2911296969","https://openalex.org/W2923014074","https://openalex.org/W2936695845","https://openalex.org/W2963748441","https://openalex.org/W2963857521","https://openalex.org/W2964223283","https://openalex.org/W2969454448","https://openalex.org/W2990138404","https://openalex.org/W3030363341","https://openalex.org/W3120043490","https://openalex.org/W3134686229","https://openalex.org/W3161027892","https://openalex.org/W3177813494","https://openalex.org/W3196850540","https://openalex.org/W4225327699","https://openalex.org/W4286892945","https://openalex.org/W4287024925","https://openalex.org/W4288057765","https://openalex.org/W4310895557","https://openalex.org/W4382704594","https://openalex.org/W4385571808","https://openalex.org/W4386908092","https://openalex.org/W4387075000","https://openalex.org/W4389215044","https://openalex.org/W4390215440","https://openalex.org/W4390306856","https://openalex.org/W4391558462","https://openalex.org/W4392011697","https://openalex.org/W4395064704","https://openalex.org/W4396566271","https://openalex.org/W4399404161","https://openalex.org/W4402671766","https://openalex.org/W4402683897","https://openalex.org/W4402716477","https://openalex.org/W4402811916","https://openalex.org/W4403324134","https://openalex.org/W6682631176","https://openalex.org/W6739901393","https://openalex.org/W6741036071","https://openalex.org/W6755207826","https://openalex.org/W6761205521","https://openalex.org/W6778883912","https://openalex.org/W6781254577","https://openalex.org/W6782879696","https://openalex.org/W6791376898","https://openalex.org/W6794686226","https://openalex.org/W6798182279","https://openalex.org/W6800166007","https://openalex.org/W6803096969","https://openalex.org/W6854348166","https://openalex.org/W6855916431","https://openalex.org/W6856592842","https://openalex.org/W6856611373","https://openalex.org/W6857804941","https://openalex.org/W6859342216","https://openalex.org/W6860005124","https://openalex.org/W6862395929","https://openalex.org/W6865177968","https://openalex.org/W6866958027","https://openalex.org/W6868863601","https://openalex.org/W6873645603"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699"],"abstract_inverted_index":{"As":[0],"machine":[1],"intelligence":[2,235],"evolves,":[3],"the":[4,10,47,182,248],"need":[5],"to":[6,26,34,123,171,180,191,238],"test":[7],"and":[8,30,74,85,113,128,208,236],"compare":[9],"problem-solving":[11],"abilities":[12],"of":[13,96],"different":[14],"AI":[15,68],"models":[16,25,48,69,157,203],"grows.":[17],"However,":[18],"current":[19,202],"benchmarks":[20,39],"are":[21,140],"often":[22,160,209],"simplistic,":[23],"allowing":[24],"perform":[27],"uniformly":[28],"well":[29],"making":[31],"it":[32,188],"difficult":[33],"distinguish":[35],"their":[36,162],"capabilities.":[37],"Additionally,":[38],"typically":[40],"rely":[41],"on":[42,185,247],"static":[43],"question-answer":[44],"pairs":[45],"that":[46,136,201],"might":[49],"memorize":[50],"or":[51],"guess.":[52],"To":[53],"address":[54],"these":[55],"limitations,":[56],"we":[57],"introduce":[58],"Dynamic":[59],"Intelligence":[60],"Assessment":[61],"(DIA),":[62],"a":[63,93,125,222,232],"novel":[64],"methodology":[65],"for":[66,225],"testing":[67],"using":[70,198],"dynamic":[71],"question":[72],"templates":[73,98],"improved":[75],"metrics":[76,122,134],"across":[77,130],"multiple":[78,131],"disciplines":[79],"such":[80],"as":[81],"mathematics,":[82],"cryptography,":[83],"cybersecurity,":[84],"computer":[86],"science.":[87],"The":[88,218,242],"accompanying":[89],"dataset,":[90],"DIA-Bench,":[91,199],"contains":[92],"diverse":[94],"collection":[95],"challenge":[97],"with":[99,205,215],"mutable":[100],"parameters":[101],"presented":[102],"in":[103,146,152],"various":[104],"formats,":[105],"including":[106],"text,":[107],"PDFs,":[108],"compiled":[109],"binaries,":[110],"visual":[111],"puzzles,":[112],"CTF-style":[114],"cybersecurity":[115],"challenges.":[116],"Our":[117],"framework":[118,220],"introduces":[119],"four":[120],"new":[121,223],"assess":[124,239],"model\u2019s":[126,233],"reliability":[127],"confidence":[129],"attempts.":[132],"These":[133],"revealed":[135],"even":[137,214],"simple":[138],"questions":[139],"frequently":[141],"answered":[142],"incorrectly":[143],"when":[144],"posed":[145],"varying":[147],"forms,":[148],"highlighting":[149],"significant":[150],"gaps":[151],"models\u2019":[153],"reliability.":[154],"Notably,":[155],"API":[156],"like":[158],"GPT-4o":[159],"overestimated":[161],"mathematical":[163],"capabilities,":[164],"while":[165],"ChatGPT-4o":[166],"demonstrated":[167],"better":[168],"performance":[169],"due":[170],"effective":[172],"tool":[173],"usage.":[174],"In":[175],"self-assessment":[176],"OpenAI\u2019s":[177],"o1-mini":[178],"proved":[179],"have":[181],"best":[183],"judgement":[184],"what":[186],"tasks":[187,207],"should":[189],"attempt":[190],"solve.":[192],"We":[193],"evaluated":[194],"25":[195],"state-of-the-art":[196],"LLMs":[197],"showing":[200],"struggle":[204],"complex":[206],"display":[210],"unexpectedly":[211],"low":[212],"confidence,":[213],"simpler":[216],"questions.":[217],"DIA":[219],"sets":[221],"standard":[224],"assessing":[226],"not":[227],"only":[228],"problem-solving,":[229],"but":[230],"also":[231],"adaptive":[234],"ability":[237],"its":[240],"limitations.":[241],"dataset":[243],"is":[244],"publicly":[245],"available":[246],"project\u2019s":[249],"page:":[250],"https://github.com/DIA-Bench.":[251]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}