{"id":"https://openalex.org/W4416610439","doi":"https://doi.org/10.52202/085713-0679","title":"MathArena: Evaluating LLMs on Uncontaminated Math Competitions","display_name":"MathArena: Evaluating LLMs on Uncontaminated Math Competitions","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4416610439","doi":"https://doi.org/10.52202/085713-0679"},"language":"en","primary_location":{"id":"doi:10.52202/085713-0679","doi":"https://doi.org/10.52202/085713-0679","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0679","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.23281","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018338894","display_name":"Mislav Balunovi\u0107","orcid":"https://orcid.org/0000-0002-7024-7688","observed_orcids":["https://orcid.org/0000-0002-7024-7688"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Mislav Balunovic","raw_affiliation_strings":["Swiss Federal Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Swiss Federal Institute of Technology","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093356429","display_name":"Jasper Dekoninck","orcid":"https://orcid.org/0009-0009-0621-170X","observed_orcids":["https://orcid.org/0009-0009-0621-170X"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Jasper Dekoninck","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081872880","display_name":"Ivo Petrov","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I58918642","display_name":"Sofia University \"St. Kliment Ohridski\"","ror":"https://ror.org/02jv3k292","country_code":"BG","type":"education","lineage":["https://openalex.org/I58918642"]}],"countries":["BG"],"is_corresponding":false,"raw_author_name":"Ivo Petrov","raw_affiliation_strings":["INSAIT, Sofia University, Bulgaria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"INSAIT, Sofia University, Bulgaria","institution_ids":["https://openalex.org/I58918642"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006417702","display_name":"Nikola M. Jovanovi\u0107","orcid":"https://orcid.org/0000-0003-0185-2304","observed_orcids":["https://orcid.org/0000-0003-0185-2304"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Nikola Jovanovi\u0107","raw_affiliation_strings":["ETH Zurich"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich","institution_ids":["https://openalex.org/I35440088"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069901599","display_name":"Martin Vechev","orcid":"https://orcid.org/0000-0002-0054-9568","observed_orcids":["https://orcid.org/0000-0002-0054-9568"]},"institutions":[{"id":"https://openalex.org/I35440088","display_name":"ETH Zurich","ror":"https://ror.org/05a28rw58","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I35440088"]},{"id":"https://openalex.org/I58918642","display_name":"Sofia University \"St. Kliment Ohridski\"","ror":"https://ror.org/02jv3k292","country_code":"BG","type":"education","lineage":["https://openalex.org/I58918642"]}],"countries":["BG","CH"],"is_corresponding":false,"raw_author_name":"Martin Vechev","raw_affiliation_strings":["ETH Zurich and INSAIT, Sofia University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ETH Zurich and INSAIT, Sofia University","institution_ids":["https://openalex.org/I35440088","https://openalex.org/I58918642"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":4.7413,"has_fulltext":true,"cited_by_count":5,"citation_normalized_percentile":{"value":0.95936221,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"22851","last_page":"22888"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5503000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.5503000020980835,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.16189999878406525,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12522","display_name":"Mathematics Education and Pedagogy","score":0.07890000194311142,"subfield":{"id":"https://openalex.org/subfields/2604","display_name":"Applied Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9200000166893005},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.9319999814033508},{"id":"https://openalex.org/keywords/mathematics-competitions","display_name":"mathematics competitions","score":0.6859999895095825},{"id":"https://openalex.org/keywords/benchmark-contamination","display_name":"benchmark contamination","score":0.6370000243186951},{"id":"https://openalex.org/keywords/memorization","display_name":"memorization","score":0.8880000114440918},{"id":"https://openalex.org/keywords/proof-writing","display_name":"proof writing","score":0.7850000262260437},{"id":"https://openalex.org/keywords/aime","display_name":"AIME","score":0.5260000228881836},{"id":"https://openalex.org/keywords/imo","display_name":"IMO","score":0.9399999976158142},{"id":"https://openalex.org/keywords/real-time-assessment","display_name":"real-time assessment","score":0.6399999856948853}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7727000117301941},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5034000277519226},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.46129998564720154},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.42419999837875366},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.33899998664855957},{"id":"https://openalex.org/C76969082","wikidata":"https://www.wikidata.org/wiki/Q486902","display_name":"Mathematical model","level":2,"score":0.3075999915599823},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.288100004196167},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.2840000092983246},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.2797999978065491},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.24500000476837158}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.52202/085713-0679","doi":"https://doi.org/10.52202/085713-0679","is_oa":false,"landing_page_url":"https://doi.org/10.52202/085713-0679","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 38","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.23281","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.23281","pdf_url":"https://arxiv.org/pdf/2505.23281","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.23281","doi":"https://doi.org/10.48550/arxiv.2505.23281","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.23281","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.23281","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.23281","pdf_url":"https://arxiv.org/pdf/2505.23281","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4416610439.pdf","grobid_xml":"https://content.openalex.org/works/W4416610439.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7160095038","https://openalex.org/W7103139038","https://openalex.org/W4416420775","https://openalex.org/W6891891353","https://openalex.org/W4403365207","https://openalex.org/W7148793925","https://openalex.org/W4410145882","https://openalex.org/W4417301601","https://openalex.org/W7118629135","https://openalex.org/W7134222339"],"abstract_inverted_index":{"The":[0],"rapid":[1],"advancement":[2],"of":[3,21,81,92,109,118,192,203],"reasoning":[4,41,134],"capabilities":[5,135],"in":[6,120,136],"large":[7],"language":[8],"models":[9,96,152,174],"(LLMs)":[10],"has":[11],"led":[12],"to":[13,38,188],"notable":[14,160],"improvements":[15],"on":[16,70,125,194],"mathematical":[17,58,204],"benchmarks.":[18],"However,":[19],"many":[20,57],"the":[22,71,107,142,190],"most":[23],"commonly":[24],"used":[25,88],"evaluation":[26,91,202],"datasets":[27],"(e.g.,":[28],"AIME":[29,121],"2024)":[30],"are":[31,54,102],"widely":[32],"available":[33],"online,":[34],"making":[35],"it":[36],"difficult":[37],"disentangle":[39],"genuine":[40],"from":[42],"potential":[43],"memorization.":[44],"Furthermore,":[45],"these":[46],"benchmarks":[47],"do":[48],"not":[49],"evaluate":[50],"proof-writing":[51,146],"capabilities,":[52],"which":[53],"crucial":[55],"for":[56,89,145,165],"tasks.":[59],"To":[60],"address":[61],"this,":[62],"we":[63,104,114,169],"introduce":[64],"MathArena,":[65],"a":[66,79],"new":[67,100],"benchmark":[68,144],"based":[69],"following":[72],"key":[73],"insight:":[74],"recurring":[75],"math":[76],"competitions":[77],"provide":[78],"stream":[80],"high-quality,":[82],"challenging":[83],"problems":[84,101],"that":[85],"can":[86],"be":[87],"real-time":[90],"LLMs.":[93],"By":[94],"evaluating":[95],"as":[97,99,129],"soon":[98],"released,":[103],"effectively":[105],"eliminate":[106],"risk":[108],"contamination.":[110],"Using":[111],"this":[112],"framework,":[113],"find":[115],"strong":[116],"signs":[117],"contamination":[119],"2024.":[122],"Nonetheless,":[123],"evaluations":[124],"harder":[126],"competitions,":[127,177,197],"such":[128],"CMIMC":[130],"2025,":[131,150],"demonstrate":[132],"impressive":[133],"top-performing":[137],"models.":[138],"MathArena":[139,185],"is":[140],"also":[141],"first":[143],"capabilities.":[147],"On":[148],"IMO":[149],"top":[151],"achieve":[153],"slightly":[154],"less":[155],"than":[156],"40%,":[157],"demonstrating":[158],"both":[159],"progress":[161,191],"and":[162,200],"significant":[163],"room":[164],"improvement.":[166],"So":[167],"far,":[168],"have":[170],"evaluated":[171],"over":[172],"$50$":[173],"across":[175],"seven":[176],"totaling":[178],"$162$":[179],"problems.":[180],"As":[181],"an":[182],"evolving":[183],"benchmark,":[184],"will":[186],"continue":[187],"track":[189],"LLMs":[193],"newly":[195],"released":[196],"ensuring":[198],"rigorous":[199],"up-to-date":[201],"reasoning.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":5}],"updated_date":"2026-10-06T10:28:23.643775","created_date":"2025-10-10T00:00:00"}