{"id":"https://openalex.org/W7116108823","doi":"https://doi.org/10.48550/arxiv.2512.14989","title":"Evaluating Large Language Models on Multimodal Chemistry Olympiad Exams","display_name":"Evaluating Large Language Models on Multimodal Chemistry Olympiad Exams","publication_year":2025,"publication_date":"2025-12-17","ids":{"openalex":"https://openalex.org/W7116108823","doi":"https://doi.org/10.48550/arxiv.2512.14989"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2512.14989","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.14989","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2512.14989","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Cui, Yiming","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cui, Yiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Yao, Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yao, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Qin, Yuxuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qin, Yuxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Xin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Xin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wang, Shijin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Shijin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Hu, Guoping","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Guoping","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5151000022888184,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5151000022888184,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.18639999628067017,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.024900000542402267,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/olympiad","display_name":"Olympiad","score":0.7684000134468079},{"id":"https://openalex.org/keywords/scientific-reasoning","display_name":"Scientific reasoning","score":0.5503000020980835},{"id":"https://openalex.org/keywords/multimodality","display_name":"Multimodality","score":0.5360999703407288},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5273000001907349},{"id":"https://openalex.org/keywords/intersection","display_name":"Intersection (aeronautics)","score":0.5199999809265137},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"Visual reasoning","score":0.42899999022483826}],"concepts":[{"id":"https://openalex.org/C130383907","wikidata":"https://www.wikidata.org/wiki/Q221956","display_name":"Olympiad","level":2,"score":0.7684000134468079},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6363999843597412},{"id":"https://openalex.org/C2992562121","wikidata":"https://www.wikidata.org/wiki/Q3817808","display_name":"Scientific reasoning","level":2,"score":0.5503000020980835},{"id":"https://openalex.org/C2780910867","wikidata":"https://www.wikidata.org/wiki/Q1952416","display_name":"Multimodality","level":2,"score":0.5360999703407288},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5273000001907349},{"id":"https://openalex.org/C64543145","wikidata":"https://www.wikidata.org/wiki/Q162942","display_name":"Intersection (aeronautics)","level":2,"score":0.5199999809265137},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.49050000309944153},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.42899999022483826},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.40400001406669617},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3239000141620636},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3190999925136566},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.29840001463890076},{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C7220189","wikidata":"https://www.wikidata.org/wiki/Q52827","display_name":"Pictogram","level":2,"score":0.2840000092983246},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.26750001311302185}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2512.14989","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.14989","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2512.14989","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2512.14989","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.6119286417961121}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"scientific":[1,122,168],"reasoning":[2,70,123],"remains":[3],"a":[4,44,144],"significant":[5],"challenge":[6],"for":[7,131,147,158],"large":[8],"language":[9],"models":[10,78],"(LLMs),":[11],"particularly":[12],"in":[13,84,95,120,139,150],"chemistry,":[14],"where":[15,83],"problem-solving":[16],"relies":[17],"on":[18,43],"symbolic":[19],"diagrams,":[20],"molecular":[21],"structures,":[22],"and":[23,33,41,68,104,112,135,154,167],"structured":[24],"visual":[25,67,105],"data.":[26],"Here,":[27],"we":[28],"systematically":[29],"evaluate":[30],"40":[31],"proprietary":[32],"open-source":[34],"multimodal":[35,137,152],"LLMs,":[36],"including":[37],"GPT-5,":[38],"o3,":[39],"Gemini-2.5-Pro,":[40],"Qwen2.5-VL,":[42],"curated":[45],"benchmark":[46,146],"of":[47,56,125,164],"Olympiad-style":[48],"chemistry":[49],"questions":[50,64],"drawn":[51],"from":[52],"over":[53],"two":[54],"decades":[55],"U.S.":[57],"National":[58],"Chemistry":[59],"Olympiad":[60],"(USNCO)":[61],"exams.":[62],"These":[63],"require":[65],"integrated":[66],"textual":[69],"across":[71],"diverse":[72],"modalities.":[73],"We":[74],"find":[75],"that":[76],"many":[77],"struggle":[79],"with":[80],"modality":[81],"fusion,":[82],"some":[85],"cases,":[86],"removing":[87],"the":[88,121,156,162],"image":[89],"even":[90],"improves":[91],"accuracy,":[92],"indicating":[93],"misalignment":[94],"vision-language":[96],"integration.":[97],"Chain-of-Thought":[98],"prompting":[99],"consistently":[100],"enhances":[101],"both":[102],"accuracy":[103],"grounding,":[106],"as":[107],"demonstrated":[108],"through":[109],"ablation":[110],"studies":[111],"occlusion-based":[113],"interpretability.":[114],"Our":[115],"results":[116],"reveal":[117],"critical":[118],"limitations":[119],"abilities":[124],"current":[126],"MLLMs,":[127],"providing":[128],"actionable":[129],"strategies":[130],"developing":[132],"more":[133],"robust":[134],"interpretable":[136],"systems":[138],"chemistry.":[140],"This":[141],"work":[142],"provides":[143],"timely":[145],"measuring":[148],"progress":[149],"domain-specific":[151],"AI":[153],"underscores":[155],"need":[157],"further":[159],"advances":[160],"at":[161],"intersection":[163],"artificial":[165],"intelligence":[166],"reasoning.":[169]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-12-19T00:00:00"}