{"id":"https://openalex.org/W4407097799","doi":"https://doi.org/10.1109/tkde.2025.3536008","title":"Are Large Language Models Really Good Logical Reasoners? A Comprehensive Evaluation and Beyond","display_name":"Are Large Language Models Really Good Logical Reasoners? A Comprehensive Evaluation and Beyond","publication_year":2025,"publication_date":"2025-02-03","ids":{"openalex":"https://openalex.org/W4407097799","doi":"https://doi.org/10.1109/tkde.2025.3536008"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2025.3536008","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2025.3536008","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084483228","display_name":"Fangzhi Xu","orcid":"https://orcid.org/0000-0003-4712-0547"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Fangzhi Xu","raw_affiliation_strings":["School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China"],"raw_orcid":"https://orcid.org/0000-0003-4712-0547","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086407377","display_name":"Qika Lin","orcid":"https://orcid.org/0000-0001-5650-0600"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qika Lin","raw_affiliation_strings":["School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China"],"raw_orcid":"https://orcid.org/0000-0001-5650-0600","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5019539533","display_name":"Jiawei Han","orcid":"https://orcid.org/0000-0002-3629-2696"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiawei Han","raw_affiliation_strings":["School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034779671","display_name":"Tianzhe Zhao","orcid":"https://orcid.org/0000-0002-2879-2703"},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianzhe Zhao","raw_affiliation_strings":["School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China"],"raw_orcid":"https://orcid.org/0000-0002-2879-2703","affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x2019;an Jiaotong University, Xi&#x2019;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]},{"raw_affiliation_string":"School of Computer Science and Technology, Ministry of Education Key Laboratory of Intelligent Networks and Network Security, Xi&#x0027;an Jiaotong University, Xi&#x0027;an, Shaanxi, China","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100361891","display_name":"Jun Liu","orcid":"https://orcid.org/0000-0002-6004-0675"},"institutions":[{"id":"https://openalex.org/I4210141776","display_name":"China XD Group (China)","ror":"https://ror.org/04ceqst84","country_code":"CN","type":"company","lineage":["https://openalex.org/I4210141776"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Liu","raw_affiliation_strings":["Shaanxi Provincial Key Laboratory of Big Data Knowledge Engineering, Xi&#x2019;an, Shaanxi, China","Shaanxi Provincial Key Laboratory of Big Data Knowledge Engineering, Xi&#x0027;an, Shaanxi, China"],"raw_orcid":"https://orcid.org/0000-0002-6004-0675","affiliations":[{"raw_affiliation_string":"Shaanxi Provincial Key Laboratory of Big Data Knowledge Engineering, Xi&#x2019;an, Shaanxi, China","institution_ids":["https://openalex.org/I4210141776"]},{"raw_affiliation_string":"Shaanxi Provincial Key Laboratory of Big Data Knowledge Engineering, Xi&#x0027;an, Shaanxi, China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100752356","display_name":"Erik Cambria","orcid":"https://orcid.org/0000-0002-3030-1280"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Erik Cambria","raw_affiliation_strings":["College of Computing and Data Science, Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-3030-1280","affiliations":[{"raw_affiliation_string":"College of Computing and Data Science, Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5084483228"],"corresponding_institution_ids":["https://openalex.org/I87445476"],"apc_list":null,"apc_paid":null,"fwci":73.8907,"has_fulltext":false,"cited_by_count":34,"citation_normalized_percentile":{"value":0.99916276,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"37","issue":"4","first_page":"1620","last_page":"1634"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8610000014305115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8610000014305115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.8557000160217285,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.8302000164985657,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8467353582382202},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.4018329083919525},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.3839070200920105}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8467353582382202},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4018329083919525},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3839070200920105}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2025.3536008","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2025.3536008","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1133573065","display_name":null,"funder_award_id":"62176207","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1661763918","display_name":null,"funder_award_id":"62293554","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2496375212","display_name":null,"funder_award_id":"62293550","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G2965804137","display_name":null,"funder_award_id":"62477037","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4303215675","display_name":null,"funder_award_id":"62192781","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4712315907","display_name":null,"funder_award_id":"62437002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4896784468","display_name":null,"funder_award_id":"62137002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6311837223","display_name":null,"funder_award_id":"62293553","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7339645210","display_name":null,"funder_award_id":"62176209","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W1533343564","https://openalex.org/W2048082592","https://openalex.org/W2072939363","https://openalex.org/W2124648060","https://openalex.org/W2130107788","https://openalex.org/W2147841362","https://openalex.org/W2971107062","https://openalex.org/W2997351066","https://openalex.org/W3034457116","https://openalex.org/W3034830866","https://openalex.org/W3153739498","https://openalex.org/W3173805051","https://openalex.org/W4205489629","https://openalex.org/W4205870266","https://openalex.org/W4221154651","https://openalex.org/W4242082217","https://openalex.org/W4243656193","https://openalex.org/W4253133923","https://openalex.org/W4284713043","https://openalex.org/W4285267480","https://openalex.org/W4289523162","https://openalex.org/W4290875505","https://openalex.org/W4290875596","https://openalex.org/W4309674289","https://openalex.org/W4320060339","https://openalex.org/W4367353919","https://openalex.org/W4385572162","https://openalex.org/W4385573405","https://openalex.org/W4389523957","https://openalex.org/W4391494845","https://openalex.org/W4392669753","https://openalex.org/W4396762144","https://openalex.org/W4404517435","https://openalex.org/W6631399359","https://openalex.org/W6771720625","https://openalex.org/W6779875681","https://openalex.org/W6786277296","https://openalex.org/W6809646742","https://openalex.org/W6810738896","https://openalex.org/W6842556415","https://openalex.org/W6846648116","https://openalex.org/W6846874006","https://openalex.org/W6848909144","https://openalex.org/W6850575147","https://openalex.org/W6851147835","https://openalex.org/W6851960618","https://openalex.org/W6852147295","https://openalex.org/W6852252866","https://openalex.org/W6852800892","https://openalex.org/W6852870998"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Logical":[0],"reasoning":[1,85,96,205,232],"consistently":[2],"plays":[3],"a":[4,26,211,226],"fundamental":[5],"and":[6,15,68,87,94,109,133,139,154,185,198,253,262,266],"significant":[7],"role":[8],"in":[9,29,72,131],"the":[10,35,43,99,162,193,203,219,260],"domains":[11],"of":[12,37,45,101,165,195,207,230,264],"knowledge":[13,196],"engineering":[14],"artificial":[16],"intelligence.":[17],"Recently,":[18],"Large":[19],"Language":[20],"Models":[21],"(LLMs)":[22],"have":[23],"emerged":[24],"as":[25],"noteworthy":[27],"innovation":[28],"natural":[30],"language":[31],"processing":[32],"(NLP).":[33],"However,":[34],"question":[36],"whether":[38],"LLMs":[39,108,265],"can":[40],"effectively":[41],"address":[42],"task":[44],"logical":[46,84,163,204,231],"reasoning,":[47],"which":[48],"requires":[49],"gradual":[50],"cognitive":[51],"inference":[52],"similar":[53],"to":[54,64,76,160,172,191],"human":[55],"intelligence,":[56],"remains":[57],"unanswered.":[58],"To":[59],"this":[60,66,73,222],"end,":[61],"we":[62,80,103,127,209],"aim":[63],"bridge":[65],"gap":[67],"provide":[69],"comprehensive":[70],"evaluations":[71,117,130],"paper.":[74],"First,":[75],"offer":[77],"systematic":[78],"evaluations,":[79,102,221],"select":[81],"fifteen":[82],"typical":[83],"datasets":[86],"organize":[88],"them":[89],"into":[90],"deductive,":[91],"inductive,":[92],"abductive":[93],"mixed-form":[95],"settings.":[97],"Considering":[98],"comprehensiveness":[100],"include":[104],"3":[105],"early-era":[106],"representative":[107],"4":[110],"trending":[111],"LLMs.":[112],"Second,":[113],"different":[114],"from":[115,176,234],"previous":[116],"relying":[118],"only":[119],"on":[120,201,218],"simple":[121],"metrics":[122],"(e.g.,":[123],"<italic":[124,142,146,150,155,180,186,238,241,244,247,250,254],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[125,143,147,151,156,181,187,239,242,245,248,251,255],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">accuracy</i>),":[126],"propose":[128,210],"fine-level":[129],"objective":[132],"subjective":[134],"manners,":[135],"covering":[136],"both":[137],"answers":[138],"explanations,":[140],"including":[141],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">answer":[144],"correctness</i>,":[145,149],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">explain":[148,152,157],"completeness</i>":[153],"redundancy</i>.":[158],"Additionally,":[159],"uncover":[161],"flaws":[164],"LLMs,":[166,208],"problematic":[167],"cases":[168],"will":[169],"be":[170],"attributed":[171],"five":[173],"error":[174],"types":[175],"two":[177],"dimensions,":[178],"i.e.,":[179],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">evidence":[182],"selection":[183],"process</i>":[184],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">reasoning":[188],"process</i>.":[189],"Third,":[190],"avoid":[192],"influences":[194],"bias":[197],"concentrate":[199],"purely":[200],"benchmarking":[202],"capability":[206,233],"new":[212],"dataset":[213],"with":[214],"neutral":[215],"content.":[216],"Based":[217],"in-depth":[220],"paper":[223],"finally":[224],"forms":[225],"general":[227],"evaluation":[228],"scheme":[229],"six":[235],"dimensions":[236],"(i.e.,":[237],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Correct</i>,":[240],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Rigorous</i>,":[243],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Self-aware</i>,":[246],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Active</i>,":[249],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Oriented</i>":[252],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">No":[256],"hallucination</i>).":[257],"It":[258],"reflects":[259],"pros":[261],"cons":[263],"gives":[267],"guiding":[268],"directions":[269],"for":[270],"future":[271],"works.":[272]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":28}],"updated_date":"2025-12-21T01:58:51.020947","created_date":"2025-10-10T00:00:00"}