{"id":"https://openalex.org/W4412888877","doi":"https://doi.org/10.18653/v1/2025.findings-acl.77","title":"LogicGame: Benchmarking Rule-Based Reasoning Abilities of Large Language Models","display_name":"LogicGame: Benchmarking Rule-Based Reasoning Abilities of Large Language Models","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412888877","doi":"https://doi.org/10.18653/v1/2025.findings-acl.77"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.77","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.77","pdf_url":"https://aclanthology.org/2025.findings-acl.77.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.77.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5049173771","display_name":"Jiayi Gui","orcid":"https://orcid.org/0000-0003-2371-4690"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiayi Gui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055597792","display_name":"Y. Liu","orcid":"https://orcid.org/0000-0003-3257-9240"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yiming Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002409683","display_name":"Jiale Cheng","orcid":"https://orcid.org/0000-0002-0032-7455"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiale Cheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102325476","display_name":"Xiaotao Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiaotao Gu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100714256","display_name":"Xiao Liu","orcid":"https://orcid.org/0000-0003-4673-8273"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085094109","display_name":"Hongning Wang","orcid":"https://orcid.org/0000-0002-6524-9195"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongning Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052284218","display_name":"Yuxiao Dong","orcid":"https://orcid.org/0000-0002-6092-2002"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuxiao Dong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044791875","display_name":"Jie Tang","orcid":"https://orcid.org/0000-0003-3487-4593"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jie Tang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5058504936","display_name":"Minlie Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Minlie Huang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1474","last_page":"1491"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9905999898910522,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9690999984741211,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9643999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8034447431564331},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7220208048820496},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.48033323884010315},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4609907269477844}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8034447431564331},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7220208048820496},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48033323884010315},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4609907269477844},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.77","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.77","pdf_url":"https://aclanthology.org/2025.findings-acl.77.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.77","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.77","pdf_url":"https://aclanthology.org/2025.findings-acl.77.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412888877.pdf","grobid_xml":"https://content.openalex.org/works/W4412888877.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W3204019825"],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"notable":[6,202],"capabilities":[7,65],"across":[8],"various":[9,198],"tasks,":[10],"showcasing":[11],"complex":[12,17,176],"problem-solving":[13],"abilities.Understanding":[14],"and":[15,28,34,43,63,88,159,192,200,213],"executing":[16],"rules,":[18],"along":[19],"with":[20,80,167],"multi-step":[21,193],"planning,":[22],"are":[23,111,157],"fundamental":[24],"to":[25,56,86,92,105,114,175,181],"logical":[26,116,207],"reasoning":[27,117,135,177,208],"critical":[29],"for":[30,129],"practical":[31],"LLM":[32],"agents":[33],"decision-making":[35],"systems.However,":[36],"evaluating":[37],"LLMs":[38,199],"as":[39],"effective":[40],"rulebased":[41],"executors":[42],"planners":[44],"remains":[45],"underexplored.In":[46],"this":[47],"paper,":[48],"we":[49,196],"introduce":[50],"LOG-ICGAME,":[51],"a":[52,76,130,148,183],"novel":[53],"benchmark":[54],"designed":[55,113],"evaluate":[57],"the":[58],"comprehensive":[59,149],"rule":[60,173,190],"understanding,":[61],"execution,":[62],"planning":[64],"of":[66,78,133,151,186],"LLMs.Unlike":[67],"traditional":[68],"benchmarks,":[69],"LOGICGAME":[70],"provides":[71],"diverse":[72],"games":[73],"that":[74],"contain":[75],"series":[77],"rules":[79],"an":[81],"initial":[82],"state,":[83],"requiring":[84],"models":[85,100],"comprehend":[87],"apply":[89],"predefined":[90,125],"regulations":[91],"solve":[93],"problems.We":[94],"create":[95],"simulated":[96],"scenarios":[97,110,166],"in":[98,179,204],"which":[99],"execute":[101],"or":[102],"plan":[103],"operations":[104],"achieve":[106],"specific":[107],"outcomes.These":[108],"game":[109,165],"specifically":[112],"distinguish":[115],"from":[118,171],"mere":[119],"knowledge":[120],"by":[121],"relying":[122],"exclusively":[123],"on":[124,189],"rules.This":[126],"separation":[127],"allows":[128],"pure":[131],"assessment":[132,150],"rule-based":[134,206],"capabilities.The":[136],"evaluation":[137,185],"considers":[138],"not":[139],"only":[140],"final":[141],"outcomes":[142],"but":[143],"also":[144],"intermediate":[145,155],"steps,":[146],"providing":[147],"model":[152,187],"performance.Moreover,":[153],"these":[154],"steps":[156],"deterministic":[158],"can":[160],"be":[161],"automatically":[162],"verified.LOGICGAME":[163],"defines":[164],"varying":[168],"difficulty":[169],"levels,":[170],"simple":[172],"applications":[174],"chains,":[178],"order":[180],"offer":[182],"precise":[184],"performance":[188],"understanding":[191],"execution.Utilizing":[194],"LOGICGAME,":[195],"test":[197],"identify":[201],"shortcomings":[203],"their":[205],"abilities.*":[209],"JG,":[210],"YL,":[211],"JC,":[212],"XG":[214],"contributed":[215],"equally.":[216]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}