{"id":"https://openalex.org/W7091379600","doi":"https://doi.org/10.5281/zenodo.17139984","title":"Evaluating Large Language Models on Multimodal Chemistry Olympiad Exams","display_name":"Evaluating Large Language Models on Multimodal Chemistry Olympiad Exams","publication_year":2025,"publication_date":"2025-12-13","ids":{"openalex":"https://openalex.org/W7091379600","doi":"https://doi.org/10.5281/zenodo.17139984"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.17139984","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17139984","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.17139984","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Cui, Yiming","orcid":"https://orcid.org/0000-0002-2452-375X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Yiming","raw_affiliation_strings":["State Key Laboratory of Cognitive Intelligence","iFLYTEK Research"],"raw_orcid":"https://orcid.org/0000-0002-2452-375X","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Intelligence","institution_ids":[]},{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yao, Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Xin","raw_affiliation_strings":["State Key Laboratory of Cognitive Intelligence","iFLYTEK Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Intelligence","institution_ids":[]},{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qin, Yuxuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Yuxuan","raw_affiliation_strings":["iFLYTEK Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xin","raw_affiliation_strings":["State Key Laboratory of Cognitive Intelligence","iFLYTEK Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Intelligence","institution_ids":[]},{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Shijin","orcid":"https://orcid.org/0000-0002-9202-7678"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijin","raw_affiliation_strings":["State Key Laboratory of Cognitive Intelligence","iFLYTEK Research"],"raw_orcid":"https://orcid.org/0000-0002-9202-7678","affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Intelligence","institution_ids":[]},{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]},{"author_position":"last","author":{"id":null,"display_name":"Hu, Guoping","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Guoping","raw_affiliation_strings":["State Key Laboratory of Cognitive Intelligence","iFLYTEK Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"State Key Laboratory of Cognitive Intelligence","institution_ids":[]},{"raw_affiliation_string":"iFLYTEK Research","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T10659","display_name":"Marine animal studies overview","score":0.14350000023841858,"subfield":{"id":"https://openalex.org/subfields/2303","display_name":"Ecology"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10659","display_name":"Marine animal studies overview","score":0.14350000023841858,"subfield":{"id":"https://openalex.org/subfields/2303","display_name":"Ecology"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12126","display_name":"Maritime Transport Emissions and Efficiency","score":0.11249999701976776,"subfield":{"id":"https://openalex.org/subfields/2305","display_name":"Environmental Engineering"},"field":{"id":"https://openalex.org/fields/23","display_name":"Environmental Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12806","display_name":"Ocean Acidification Effects and Responses","score":0.0494999997317791,"subfield":{"id":"https://openalex.org/subfields/1910","display_name":"Oceanography"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/olympiad","display_name":"Olympiad","score":0.7702000141143799},{"id":"https://openalex.org/keywords/metadata","display_name":"Metadata","score":0.695900022983551},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.5194000005722046},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4487000107765198},{"id":"https://openalex.org/keywords/intersection","display_name":"Intersection (aeronautics)","score":0.4350000023841858},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.34310001134872437}],"concepts":[{"id":"https://openalex.org/C130383907","wikidata":"https://www.wikidata.org/wiki/Q221956","display_name":"Olympiad","level":2,"score":0.7702000141143799},{"id":"https://openalex.org/C93518851","wikidata":"https://www.wikidata.org/wiki/Q180160","display_name":"Metadata","level":2,"score":0.695900022983551},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.683899998664856},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.5194000005722046},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4487000107765198},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.4350000023841858},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4138999879360199},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.34310001134872437},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3285999894142151},{"id":"https://openalex.org/C68762167","wikidata":"https://www.wikidata.org/wiki/Q910164","display_name":"Cheminformatics","level":2,"score":0.326200008392334},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2955999970436096},{"id":"https://openalex.org/C45874996","wikidata":"https://www.wikidata.org/wiki/Q37045","display_name":"Markup language","level":3,"score":0.29269999265670776},{"id":"https://openalex.org/C2780660688","wikidata":"https://www.wikidata.org/wiki/Q25052564","display_name":"Multimodal learning","level":2,"score":0.29100000858306885},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.2694999873638153},{"id":"https://openalex.org/C2992562121","wikidata":"https://www.wikidata.org/wiki/Q3817808","display_name":"Scientific reasoning","level":2,"score":0.26910001039505005},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.26589998602867126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.17139984","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17139984","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.17139984","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.17139984","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Overview":[0],"This":[1,166,196],"repository":[2,197],"contains":[3,198],"resources":[4],"for":[5,32,156,172,183,204,232,244,256,267,281,440,547,554],"our":[6,205],"paper":[7],"\"Evaluating":[8],"large":[9,33],"language":[10,34],"models":[11,35,103],"on":[12,43,68],"multimodal":[13,60,162,177,207],"chemistry":[14,74,208],"olympiad":[15],"exams\"":[16],"published":[17],"at":[18,186],"Communications":[19],"Chemistry.":[20],"Paper":[21],"link:":[22],"https://www.nature.com/articles/s42004-025-01782-x":[23],"Abstract":[24],"Multimodal":[25],"scientific":[26,147,193],"reasoning":[27,95,148],"remains":[28],"a":[29,69,169,313,530,563,644,668],"significant":[30],"challenge":[31],"(LLMs),":[36],"particularly":[37],"in":[38,109,120,145,164,175,359,370,420,443,494,505,538,670,707],"chemistry,":[39],"where":[40,108],"problem-solving":[41],"relies":[42],"symbolic":[44],"diagrams,":[45],"molecular":[46],"structures,":[47],"and":[48,58,66,93,129,137,160,179,192,201,413,426,533,601,605],"structured":[49],"visual":[50,92,130],"data.":[51],"Here,":[52],"we":[53],"systematically":[54],"evaluate":[55],"40":[56],"proprietary":[57],"open-source":[59],"LLMs,":[61],"including":[62,423],"GPT-5,":[63],"o3,":[64],"Gemini-2.5-Pro,":[65],"Qwen2.5-VL,":[67],"curated":[70],"benchmark":[71,171],"of":[72,81,150,189,344,379,410,432,591,632],"Olympiad-style":[73],"questions":[75,89,236,248,442,633],"drawn":[76],"from":[77,220],"over":[78],"two":[79],"decades":[80],"U.S.":[82],"National":[83],"Chemistry":[84],"Olympiad":[85],"(USNCO)":[86],"exams.":[87],"These":[88],"require":[90],"integrated":[91],"textual":[94],"across":[96],"diverse":[97],"modalities.":[98],"We":[99,561],"find":[100],"that":[101,608,656],"many":[102],"struggle":[104],"with":[105,620],"modality":[106],"fusion,":[107],"some":[110],"cases,":[111],"removing":[112],"the":[113,146,181,187,199,345,360,411,485,500,506,511,519,527,539,571,576,592,616,637,711],"image":[114,366,478],"even":[115],"improves":[116],"accuracy,":[117],"indicating":[118,629],"misalignment":[119],"vision-language":[121],"integration.":[122],"Chain-of-Thought":[123],"prompting":[124],"consistently":[125],"enhances":[126],"both":[127],"accuracy":[128],"grounding,":[131],"as":[132,529,536,667],"demonstrated":[133],"through":[134],"ablation":[135],"studies":[136],"occlusion-based":[138],"interpretability.":[139],"Our":[140],"results":[141],"reveal":[142],"critical":[143],"limitations":[144],"abilities":[149],"current":[151],"MLLMs,":[152],"providing":[153],"actionable":[154],"strategies":[155],"developing":[157],"more":[158],"robust":[159],"interpretable":[161],"systems":[163],"chemistry.":[165],"work":[167],"provides":[168],"timely":[170],"measuring":[173],"progress":[174],"domain-specific":[176],"AI":[178],"underscores":[180],"need":[182],"further":[184],"advances":[185],"intersection":[188],"artificial":[190],"intelligence":[191],"reasoning.":[194],"README":[195],"metadata":[200,231,243,481,621],"evaluation":[202,282,292,638],"script":[203,293,298,560,565,617,639],"USNCO\u2011V":[206],"benchmark.":[209],"No":[210],"copyrighted":[211],"text":[212],"or":[213,351,372,525,543,603,627],"images":[214,260,271],"are":[215],"distributed\u2014readers":[216],"manually":[217],"crop":[218],"diagrams":[219],"publicly":[221],"available":[222,439],"USNCO":[223,338],"PDFs.":[224],"Repository":[225],"structure":[226],".":[227,584],"\u251c\u2500\u2500":[228,240,252,263,274,284,289,294],"usnco-v_local.json":[229],"#":[230,242,254,265,276,291,296,306],"all":[233,245,257,268,548],"204":[234],"local":[235,258,330,586],"(including":[237,249],"2025":[238,250],"exam)":[239,251],"usnco-v_national.json":[241],"269":[246],"national":[247,269,441],"image-local/":[253],"folder":[255,266,374,541],"question":[259,270,323,356,396,412,501],"(not":[261,272],"included)":[262,273],"image-national/":[264],"sample_output/":[275],"sample":[277,297,564,702],"model":[278,594,641],"output":[279,643,703],"files":[280],"\u2502":[283,286],"predictions-local.json":[285],"\u2514\u2500\u2500":[287,304],"predictions-national.json":[288],"evaluate.py":[290,715],"generate_qwen25vl.py":[295],"to":[299,337,474,567,634,650,657],"generate":[300,568],"predictions":[301,569,663],"using":[302,518,570],"Qwen2.5-VL":[303,572,593],"README.md":[305],"this":[307],"file":[308,646,704],"Metadata":[309],"format":[310],"usnco_v_meta.json":[311],"is":[312,705],"JSON":[314,645],"array;":[315],"each":[316,476,480,648],"entry":[317],"has:":[318],"-":[319,327,333,340,347,353,362,375,385,395,400,405,415,428,588,595,623],"qid":[320,649],"(string):":[321,329,335,349,364,377,387,397,402,407],"unique":[322],"ID":[324],"(e.g.":[325],"\"2025-12\")":[326],"pdf":[328],"PDF":[331,339,486,496],"filename":[332],"pdf_url":[334],"URL":[336],"year":[341,343],"(int):":[342,355],"exam":[346],"type":[348],"\"local\"":[350],"\"national\"":[352],"original_qid":[354],"number":[357],"printed":[358],"margin":[361],"image_id":[363],"corresponding":[365,540],"name":[367],"(without":[368],".png)":[369],"`image-local`":[371],"`image-national`":[373],"image_type":[376],"one":[378],"five":[380],"categories":[381,419],"(\"table\",":[382],"\"chart\",":[383],"\u2026)":[384],"answer":[386],"correct":[388],"option":[389,653],"letter":[390],"(\"A\",":[391],"\"B\",":[392,695],"\"C\",":[393,679,683,684],"\"D\")":[394],"empty":[398,403],"(placeholder)":[399,404],"choices":[401,414],"summary":[406,409],"brief":[408],"category":[416],"(dict):":[417],"knowledge":[418],"three":[421],"hierarchies,":[422],"\"main_category\",":[424],"\"subcategory\",":[425],"\"skill_or_focus\"":[427],"human_rate":[429],"(float):":[430],"percentage":[431],"students":[433],"who":[434],"answered":[435],"correctly":[436],"(0-100),":[437],"only":[438],"certain":[444],"years":[445],"Prerequisites":[446],"1.":[447,483],"**Python":[448],"3.10+**":[449],"2.":[450,491],"**Dependencies**":[451],"pip":[452,456,460,466,470],"install":[453,457,461,467,471],"-U":[454],"transformers":[455],"huggingface_hub":[458],"hf-transfer":[459],"accelerate":[462],"bitsandbytes":[463],"peft":[464],"datasets":[465],"flash-attn":[468],"--no-build-isolation":[469],"prettytable":[472],"How":[473],"obtain":[475],"diagram":[477,512],"For":[479,672],"entry:":[482],"Download":[484],"wget":[487],"\"<pdf_url>\"":[488],"-O":[489],"\"<pdf>\"":[490],"Open":[492],"<pdf>":[493],"your":[495],"viewer.":[497],"3.":[498],"Locate":[499],"block":[502],"labeled":[503],"`original_qid`":[504],"left":[507],"margin.":[508],"4.":[509],"Select":[510],"(and":[513],"its":[514,651],"caption,":[515],"if":[516],"any)":[517],"viewer\u2019s":[520],"rectangular\u2010selection":[521],"tool.":[522],"5.":[523],"Copy":[524],"export":[526],"selection":[528],"PNG":[531],"image,":[532],"save":[534],"it":[535],"`{image_id}.png`":[537],"(`image-local`":[542],"`image-national`)":[544],"6.":[545],"Repeat":[546],"questions.":[549],"(Approximately":[550],"1-2":[551],"hours":[552],"total":[553],"473":[555],"questions.)":[556],"Running":[557,636],"thep":[558],"prediction":[559],"provide":[562],"`generate_qwenvl25.py`":[566],"model.":[573],"To":[574,709],"run":[575,619,710],"script,":[577],"use:":[578,713],"python":[579,714],"generate_qwenvl25.py":[580],"--model_dir":[581],"qwen2.5-vl-3b-inst":[582],"--data_dir":[583],"--mode":[585],"where,":[587],"`--model_dir`:":[589],"directory":[590,597],"`--data_dir`:":[596],"containing":[598],"either":[599,625],"`usnco-v_local.json`":[600],"`image-local/`,":[602],"`usnco-v_national.json`":[604],"`image-national/`.":[606],"Note":[607,655],"you":[609],"must":[610],"curate":[611],"complete":[612],"dataset":[613],"before":[614],"running":[615],"(cannot":[618],"only).":[622],"`--mode`:":[624],"`local`":[626],"`national`,":[628],"which":[630],"set":[631],"use.":[635],"Your":[640],"should":[642,664],"mapping":[647],"predicted":[652],"letters.":[654],"allow":[658],"calculate":[659],"pass@5,":[660],"consistency@5,":[661],"multiple":[662],"be":[665],"stored":[666],"list":[669],"`prediction`.":[671],"example:":[673],"[":[674],"{":[675,687],"\"qid\":":[676,688],"\"2025-12\",":[677],"\"answer\":":[678,690],"\"prediction\":":[680,692],"[\"C\",":[681],"\"A\",":[682,691,694,696],"\"C\"]":[685],"},":[686,698],"\"2025-20\",":[689],"[\"A\",":[693],"\"A\"]":[697],"...":[699],"]":[700],"A":[701],"provided":[706],"`sample_output/predictions-*.json`.":[708],"evaluation,":[712],"sample_output/predictions-local.json":[716]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-16T00:00:00"}