{"id":"https://openalex.org/W4407570748","doi":"https://doi.org/10.48550/arxiv.2502.08680","title":"Mathematical Reasoning in Large Language Models: Assessing Logical and Arithmetic Errors across Wide Numerical Ranges","display_name":"Mathematical Reasoning in Large Language Models: Assessing Logical and Arithmetic Errors across Wide Numerical Ranges","publication_year":2025,"publication_date":"2025-02-12","ids":{"openalex":"https://openalex.org/W4407570748","doi":"https://doi.org/10.48550/arxiv.2502.08680"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.08680","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.08680","pdf_url":"https://arxiv.org/pdf/2502.08680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.08680","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116269531","display_name":"Safal Shrestha","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Shrestha, Safal","raw_affiliation_strings":["New York University Abu Dhabi"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University Abu Dhabi","institution_ids":["https://openalex.org/I120250893"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109709876","display_name":"Minwu Kim","orcid":"https://orcid.org/0009-0005-8104-2142","observed_orcids":["https://orcid.org/0009-0005-8104-2142"]},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Kim, Minwu","raw_affiliation_strings":["New York University Abu Dhabi"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University Abu Dhabi","institution_ids":["https://openalex.org/I120250893"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101801953","display_name":"Keith W. Ross","orcid":"https://orcid.org/0000-0003-2556-805X","observed_orcids":["https://orcid.org/0000-0003-2556-805X"]},"institutions":[{"id":"https://openalex.org/I120250893","display_name":"New York University Abu Dhabi","ror":"https://ror.org/00e5k0821","country_code":"AE","type":"education","lineage":["https://openalex.org/I120250893","https://openalex.org/I57206974"]}],"countries":["AE"],"is_corresponding":false,"raw_author_name":"Ross, Keith","raw_affiliation_strings":["New York University Abu Dhabi"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University Abu Dhabi","institution_ids":["https://openalex.org/I120250893"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I120250893"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7294999957084656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7294999957084656,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04859999939799309,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.03519999980926514,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.8370000123977661},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9990000128746033},{"id":"https://openalex.org/keywords/gsm8k","display_name":"GSM8K","score":0.9649999737739563},{"id":"https://openalex.org/keywords/numerical-range","display_name":"numerical range","score":0.41100001335144043},{"id":"https://openalex.org/keywords/logical-errors","display_name":"logical errors","score":0.7910000085830688},{"id":"https://openalex.org/keywords/calculation-errors","display_name":"calculation errors","score":0.9929999709129333},{"id":"https://openalex.org/keywords/word-problems","display_name":"word problems","score":0.7369999885559082},{"id":"https://openalex.org/keywords/dataset-construction","display_name":"dataset construction","score":0.44999998807907104}],"concepts":[{"id":"https://openalex.org/C94375191","wikidata":"https://www.wikidata.org/wiki/Q11205","display_name":"Arithmetic","level":1,"score":0.6690990328788757},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5606529712677002},{"id":"https://openalex.org/C43971567","wikidata":"https://www.wikidata.org/wiki/Q3142865","display_name":"Logical reasoning","level":2,"score":0.4281332492828369},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3242949843406677},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.18701288104057312}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.08680","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.08680","pdf_url":"https://arxiv.org/pdf/2502.08680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.08680","doi":"https://doi.org/10.48550/arxiv.2502.08680","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.08680","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.08680","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.08680","pdf_url":"https://arxiv.org/pdf/2502.08680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320310501","display_name":"New York University Shanghai","ror":"https://ror.org/02vpsdb40"},{"id":"https://openalex.org/F4320319918","display_name":"York University","ror":"https://ror.org/05fq50484"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4407570748.pdf","grobid_xml":"https://content.openalex.org/works/W4407570748.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4403324134","https://openalex.org/W4407719116","https://openalex.org/W4415055548","https://openalex.org/W4402671761","https://openalex.org/W6912308223","https://openalex.org/W4417224588","https://openalex.org/W4417299031","https://openalex.org/W4415329307","https://openalex.org/W7161833538","https://openalex.org/W4411119841"],"abstract_inverted_index":{"Mathematical":[0],"reasoning":[1,39,90,120,155],"in":[2,59,104,119,166],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"is":[7],"often":[8],"evaluated":[9],"using":[10],"benchmarks":[11],"with":[12,97,121],"limited":[13],"numerical":[14,57,68,112,123,164],"ranges,":[15],"failing":[16],"to":[17,33,62,108],"reflect":[18],"real-world":[19],"problem-solving":[20],"across":[21,66],"diverse":[22],"scales.":[23,69],"Furthermore,":[24],"most":[25],"existing":[26],"evaluation":[27,88,151],"methods":[28],"only":[29],"compare":[30],"model":[31,64],"outputs":[32],"ground-truth":[34],"answers,":[35],"obscuring":[36],"insights":[37],"into":[38],"processes.":[40],"To":[41],"address":[42],"these":[43],"limitations,":[44],"we":[45,71],"introduce":[46],"GSM-Ranges,":[47],"a":[48,73,85,101,116,149],"dataset":[49],"generator":[50],"derived":[51],"from":[52],"GSM8K":[53],"that":[54,77],"systematically":[55],"perturbs":[56],"values":[58],"math":[60],"problems":[61],"assess":[63],"robustness":[65],"varying":[67],"Additionally,":[70],"propose":[72],"novel":[74],"grading":[75],"methodology":[76],"distinguishes":[78],"between":[79],"logical":[80,105],"and":[81,157],"non-logical":[82],"errors,":[83],"offering":[84],"more":[86],"precise":[87],"of":[89,152],"processes":[91],"beyond":[92],"computational":[93],"accuracy.":[94],"Our":[95],"experiments":[96],"various":[98],"models":[99,127],"reveal":[100],"significant":[102],"increase":[103],"error":[106],"rates-up":[107],"14":[109],"percentage":[110],"points-as":[111],"complexity":[113],"rises,":[114],"demonstrating":[115],"general":[117],"weakness":[118],"out-of-distribution":[122],"values.":[124],"Moreover,":[125],"while":[126],"demonstrate":[128],"high":[129],"accuracy":[130],"on":[131],"standalone":[132],"arithmetic":[133],"tasks,":[134],"their":[135],"performance":[136],"deteriorates":[137],"substantially":[138],"when":[139],"computations":[140],"are":[141],"embedded":[142],"within":[143],"word":[144],"problems.":[145],"These":[146],"findings":[147],"provide":[148],"comprehensive":[150],"LLMs'":[153],"mathematical":[154],"capabilities":[156],"inform":[158],"future":[159],"research":[160],"directions":[161],"for":[162],"improving":[163],"generalization":[165],"language":[167],"models.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-09-30T09:27:27.532063","created_date":"2025-02-15T00:00:00"}