{"id":"https://openalex.org/W7167111373","doi":"https://doi.org/10.48550/arxiv.2607.00436","title":"PHREEQC-MCQ-200: A Diagnostic Benchmark for Tool-Augmented Scientific Simulator Agents","display_name":"PHREEQC-MCQ-200: A Diagnostic Benchmark for Tool-Augmented Scientific Simulator Agents","publication_year":2026,"publication_date":"2026-07-01","ids":{"openalex":"https://openalex.org/W7167111373","doi":"https://doi.org/10.48550/arxiv.2607.00436"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.00436","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.00436","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.00436","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139872569","display_name":"Ke Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Ke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093837552","display_name":"Sahchit Chundur","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chundur, Sahchit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139863723","display_name":"Mohammad Javad Qomi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qomi, Mohammad Javad","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139890003","display_name":"Maziar Raissi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raissi, Maziar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.4814000129699707,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.4814000129699707,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.38749998807907104,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.005100000184029341,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8094000220298767},{"id":"https://openalex.org/keywords/protocol","display_name":"Protocol (science)","score":0.6032000184059143},{"id":"https://openalex.org/keywords/commit","display_name":"Commit","score":0.6017000079154968},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.5863000154495239},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5623999834060669},{"id":"https://openalex.org/keywords/interface","display_name":"Interface (matter)","score":0.4422999918460846},{"id":"https://openalex.org/keywords/aggregate","display_name":"Aggregate (composite)","score":0.42669999599456787},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.41769999265670776}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8094000220298767},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7634999752044678},{"id":"https://openalex.org/C2780385302","wikidata":"https://www.wikidata.org/wiki/Q367158","display_name":"Protocol (science)","level":3,"score":0.6032000184059143},{"id":"https://openalex.org/C153180980","wikidata":"https://www.wikidata.org/wiki/Q19776675","display_name":"Commit","level":2,"score":0.6017000079154968},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.5863000154495239},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5623999834060669},{"id":"https://openalex.org/C113843644","wikidata":"https://www.wikidata.org/wiki/Q901882","display_name":"Interface (matter)","level":4,"score":0.4422999918460846},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.42669999599456787},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.41769999265670776},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3889000117778778},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.3630000054836273},{"id":"https://openalex.org/C126042441","wikidata":"https://www.wikidata.org/wiki/Q1324888","display_name":"Frame (networking)","level":2,"score":0.35929998755455017},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3450999855995178},{"id":"https://openalex.org/C171268870","wikidata":"https://www.wikidata.org/wiki/Q1486676","display_name":"GRASP","level":2,"score":0.3425999879837036},{"id":"https://openalex.org/C94966114","wikidata":"https://www.wikidata.org/wiki/Q29256","display_name":"Black box","level":2,"score":0.304500013589859},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.303600013256073},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.2994000017642975},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2969000041484833},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.28870001435279846},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.2883000075817108},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.25450000166893005},{"id":"https://openalex.org/C138379479","wikidata":"https://www.wikidata.org/wiki/Q1116876","display_name":"Scientific modelling","level":2,"score":0.2529999911785126}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.00436","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.00436","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.00436","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.00436","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"model":[2,73],"agents":[3,35,53,98,173],"are":[4,94],"increasingly":[5],"connected":[6],"to":[7,54,65],"scientific":[8,18,153,172],"software,":[9],"yet":[10],"it":[11,137],"remains":[12],"unclear":[13],"when":[14],"tool":[15,154],"access":[16,76],"makes":[17],"computation":[19,190],"more":[20,25],"reliable":[21],"rather":[22,161],"than":[23,162],"merely":[24],"complex.":[26],"We":[27,114,167],"introduce":[28],"PHREEQC-MCQ-200,":[29],"a":[30,163],"benchmark":[31,41],"for":[32,87,133,140],"evaluating":[33],"tool-augmented":[34,97],"on":[36],"deterministic":[37],"aqueous-geochemistry":[38],"simulations.":[39],"The":[40],"contains":[42],"200":[43],"multiple-choice":[44],"questions":[45],"derived":[46],"from":[47],"21":[48],"validated":[49],"PHREEQC":[50],"scenarios,":[51],"requiring":[52],"construct":[55],"simulator":[56,75,148],"inputs,":[57],"execute":[58],"PHREEQC,":[59],"inspect":[60],"structured":[61,147],"outputs,":[62],"and":[63,71,187],"commit":[64],"final":[66],"answers.":[67],"Across":[68],"multiple":[69],"frontier":[70],"mid-tier":[72,141],"families,":[74],"substantially":[77],"improves":[78],"aggregate":[79],"accuracy,":[80,178],"confirming":[81],"that":[82,109,117,143,169],"grounded":[83],"execution":[84],"is":[85],"necessary":[86],"many":[88],"scientific-computation":[89],"tasks.":[90],"However,":[91],"the":[92,189],"gains":[93],"not":[95,176],"monotonic:":[96],"also":[99,180],"lose":[100],"items":[101],"they":[102],"answered":[103],"correctly":[104],"without":[105],"tools,":[106],"revealing":[107],"regressions":[108],"average":[110],"accuracy":[111,132],"alone":[112],"hides.":[113],"further":[115],"show":[116],"output-access":[118,183],"protocol":[119],"matters.":[120],"A":[121],"table-of-contents":[122],"interface":[123],"can":[124],"reduce":[125],"token":[126],"cost":[127],"while":[128],"preserving":[129],"or":[130],"improving":[131],"stronger":[134],"models,":[135],"but":[136,179],"degrades":[138],"performance":[139],"models":[142],"cannot":[144],"reliably":[145],"navigate":[146],"outputs.":[149],"PHREEQC-MCQ-200":[150],"therefore":[151],"frames":[152],"use":[155],"as":[156],"an":[157],"end-to-end":[158],"diagnostic":[159],"problem":[160],"simple":[164],"tool-calling":[165],"capability.":[166],"argue":[168],"evaluations":[170],"of":[171],"should":[174],"report":[175],"only":[177],"item-level":[181],"retention,":[182],"sensitivity,":[184],"trajectory":[185],"failures,":[186],"where":[188],"chain":[191],"breaks.":[192]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-03T00:00:00"}