{"id":"https://openalex.org/W4416003953","doi":"https://doi.org/10.1145/3731599.3767405","title":"Beyond End-to-End: Understanding the Limits of LLMs in Scientific Problem Solving","display_name":"Beyond End-to-End: Understanding the Limits of LLMs in Scientific Problem Solving","publication_year":2025,"publication_date":"2025-11-07","ids":{"openalex":"https://openalex.org/W4416003953","doi":"https://doi.org/10.1145/3731599.3767405"},"language":null,"primary_location":{"id":"doi:10.1145/3731599.3767405","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767405","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5041416337","display_name":"Youyuan Liu","orcid":"https://orcid.org/0009-0002-6353-3659"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Youyuan Liu","raw_affiliation_strings":["Temple University, Philadelphia, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0009-0002-6353-3659","affiliations":[{"raw_affiliation_string":"Temple University, Philadelphia, Pennsylvania, USA","institution_ids":["https://openalex.org/I84392919"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103124363","display_name":"Sheng Di","orcid":"https://orcid.org/0000-0002-9935-5674"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sheng Di","raw_affiliation_strings":["Argonne National Laboratory (ANL), Lemont, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0002-9935-5674","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory (ANL), Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045490083","display_name":"Neil Getty","orcid":"https://orcid.org/0000-0003-0535-8362"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Neil Getty","raw_affiliation_strings":["Argonne National Laboratory (ANL), Lemont, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0003-0535-8362","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory (ANL), Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037238294","display_name":"Tanwi Mallick","orcid":"https://orcid.org/0000-0003-3716-4693"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tanwi Mallick","raw_affiliation_strings":["Argonne National Laboratory (ANL), Lemont, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0003-3716-4693","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory (ANL), Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056940311","display_name":"Robert Underwood","orcid":"https://orcid.org/0000-0002-1464-729X"},"institutions":[{"id":"https://openalex.org/I1282105669","display_name":"Argonne National Laboratory","ror":"https://ror.org/05gvnxz63","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282105669","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I40347166"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Robert Underwood","raw_affiliation_strings":["Argonne National Laboratory (ANL), Lemont, Illinois, USA"],"raw_orcid":"https://orcid.org/0000-0002-1464-729X","affiliations":[{"raw_affiliation_string":"Argonne National Laboratory (ANL), Lemont, Illinois, USA","institution_ids":["https://openalex.org/I1282105669"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103201662","display_name":"Sian Jin","orcid":"https://orcid.org/0009-0009-9250-0611"},"institutions":[{"id":"https://openalex.org/I84392919","display_name":"Temple University","ror":"https://ror.org/00kx1jb78","country_code":"US","type":"education","lineage":["https://openalex.org/I84392919"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sian Jin","raw_affiliation_strings":["Temple University, Philadelphia, Pennsylvania, USA"],"raw_orcid":"https://orcid.org/0009-0009-9250-0611","affiliations":[{"raw_affiliation_string":"Temple University, Philadelphia, Pennsylvania, USA","institution_ids":["https://openalex.org/I84392919"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"588","last_page":"593"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9598000049591064,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9598000049591064,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.016100000590085983,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.002300000051036477,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.6983000040054321},{"id":"https://openalex.org/keywords/scientific-reasoning","display_name":"Scientific reasoning","score":0.4133000075817108},{"id":"https://openalex.org/keywords/question-answering","display_name":"Question answering","score":0.4034999907016754},{"id":"https://openalex.org/keywords/scientific-modelling","display_name":"Scientific modelling","score":0.37869998812675476},{"id":"https://openalex.org/keywords/scientific-discovery","display_name":"Scientific discovery","score":0.36660000681877136},{"id":"https://openalex.org/keywords/visualization","display_name":"Visualization","score":0.3447999954223633},{"id":"https://openalex.org/keywords/visual-approach","display_name":"Visual approach","score":0.301800012588501}],"concepts":[{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.6983000040054321},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6424999833106995},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5177000164985657},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.41589999198913574},{"id":"https://openalex.org/C2992562121","wikidata":"https://www.wikidata.org/wiki/Q3817808","display_name":"Scientific reasoning","level":2,"score":0.4133000075817108},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.4034999907016754},{"id":"https://openalex.org/C138379479","wikidata":"https://www.wikidata.org/wiki/Q1116876","display_name":"Scientific modelling","level":2,"score":0.37869998812675476},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3671000003814697},{"id":"https://openalex.org/C2984917352","wikidata":"https://www.wikidata.org/wiki/Q12772819","display_name":"Scientific discovery","level":2,"score":0.36660000681877136},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.3447999954223633},{"id":"https://openalex.org/C2777055276","wikidata":"https://www.wikidata.org/wiki/Q7936580","display_name":"Visual approach","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C2777655017","wikidata":"https://www.wikidata.org/wiki/Q1501161","display_name":"Toolbox","level":2,"score":0.2847999930381775},{"id":"https://openalex.org/C195732255","wikidata":"https://www.wikidata.org/wiki/Q981008","display_name":"Sociology of scientific knowledge","level":2,"score":0.28189998865127563},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.2775999903678894},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2533000111579895}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3731599.3767405","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3731599.3767405","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '25 Workshops of the International Conference for High Performance Computing, Networking, Storage and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2690523999","display_name":null,"funder_award_id":"ARL W911NF212007","funder_id":"https://openalex.org/F4320338295","funder_display_name":"Army Research Laboratory"}],"funders":[{"id":"https://openalex.org/F4320338295","display_name":"Army Research Laboratory","ror":"https://ror.org/011hc8f90"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W3138054712","https://openalex.org/W4399208468","https://openalex.org/W4399730268","https://openalex.org/W4406157371","https://openalex.org/W4411523078"],"related_works":[],"abstract_inverted_index":{"Multimodal":[0],"large":[1,114],"language":[2],"models":[3,38,88],"(MLLMs)":[4],"are":[5,29,79],"now":[6],"widely":[7],"used":[8],"across":[9],"many":[10],"applications,":[11],"including":[12],"scientific":[13,51,143],"question":[14,52],"answering":[15,53],"that":[16,49,76,123,129],"requires":[17],"combining":[18],"visual":[19,72],"and":[20,66,127],"textual":[21],"inputs.":[22],"However,":[23],"existing":[24],"benchmarks":[25,122],"in":[26,110],"this":[27,42],"area":[28],"mostly":[30],"end-to-end,":[31],"making":[32],"it":[33],"difficult":[34],"to":[35,97,106,137],"pinpoint":[36],"where":[37],"fail.":[39],"To":[40],"address":[41],"gap,":[43],"we":[44],"design":[45],"an":[46],"evaluation":[47],"framework":[48],"decomposes":[50],"into":[54],"subtasks":[55],"for":[56],"fine-grained":[57],"assessment.":[58],"We":[59],"evaluate":[60],"two":[61],"MLLMs,":[62],"Gemini":[63],"2.5":[64],"Pro":[65],"Qwen2.5-VL-32B-Instruct,":[67],"on":[68,141],"questions":[69],"involving":[70],"high-resolution":[71],"data.":[73],"Results":[74],"show":[75],"accurate":[77],"answers":[78],"unattainable":[80],"without":[81],"scripting":[82],"or":[83,99,131],"tool":[84],"use.":[85],"Although":[86],"both":[87],"can":[89],"solve":[90],"individual":[91],"subtasks,":[92],"such":[93],"as":[94],"mapping":[95],"cities":[96],"coordinates":[98],"computing":[100],"pixel":[101],"positions,":[102],"they":[103],"often":[104],"fail":[105],"integrate":[107],"these":[108],"abilities":[109],"end-to-end":[111],"reasoning,":[112],"producing":[113],"deviations.":[115],"Our":[116],"findings":[117],"highlight":[118],"the":[119],"importance":[120],"of":[121],"expose":[124],"reasoning":[125],"bottlenecks":[126],"suggest":[128],"agent-based":[130],"multi-model":[132],"approaches":[133],"may":[134],"be":[135],"required":[136],"achieve":[138],"reliable":[139],"performance":[140],"complex":[142],"tasks.":[144]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-11-07T00:00:00"}