{"id":"https://openalex.org/W4387323979","doi":"https://doi.org/10.48550/arxiv.2310.00836","title":"Towards LogiGLUE: A Brief Survey and A Benchmark for Analyzing Logical Reasoning Capabilities of Language Models","display_name":"Towards LogiGLUE: A Brief Survey and A Benchmark for Analyzing Logical Reasoning Capabilities of Language Models","publication_year":2023,"publication_date":"2023-10-02","ids":{"openalex":"https://openalex.org/W4387323979","doi":"https://doi.org/10.48550/arxiv.2310.00836"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.00836","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.00836","pdf_url":"https://arxiv.org/pdf/2310.00836","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.00836","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101358053","display_name":"Man Luo","orcid":"https://orcid.org/0009-0008-4395-5049","observed_orcids":["https://orcid.org/0009-0008-4395-5049"]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Luo, Man","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109730218","display_name":"Shrinidhi Kumbhar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kumbhar, Shrinidhi","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":null,"display_name":"shen, Ming","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"shen, Ming","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106524687","display_name":"Mihir Parmar","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Parmar, Mihir","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004217423","display_name":"Neeraj Varshney","orcid":"https://orcid.org/0000-0003-0752-2103","observed_orcids":["https://orcid.org/0000-0003-0752-2103"]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Varshney, Neeraj","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017986865","display_name":"Pratyay Banerjee","orcid":"https://orcid.org/0000-0001-5634-410X","observed_orcids":["https://orcid.org/0000-0001-5634-410X"]},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Banerjee, Pratyay","raw_affiliation_strings":["Amazon Alexa AI , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Amazon Alexa AI , USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071844229","display_name":"Somak Aditya","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I145894827","display_name":"Indian Institute of Technology Kharagpur","ror":"https://ror.org/03w5sq511","country_code":"IN","type":"education","lineage":["https://openalex.org/I145894827"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Aditya, Somak","raw_affiliation_strings":["IIT KGP , India"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IIT KGP , India","institution_ids":["https://openalex.org/I145894827"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083735830","display_name":"Chitta R. Baral","orcid":"https://orcid.org/0000-0002-7549-723X","observed_orcids":["https://orcid.org/0000-0002-7549-723X"]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Baral, Chitta","raw_affiliation_strings":["School of Computing and Augmented Intelligence Arizona State University , Tempe , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing and Augmented Intelligence Arizona State University , Tempe , USA","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.6518999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11010","display_name":"Logic, Reasoning, and Knowledge","score":0.6518999934196472,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.28690001368522644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.02459999918937683,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/logical-reasoning","display_name":"logical reasoning","score":0.9160000085830688},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.984000027179718},{"id":"https://openalex.org/keywords/deductive-reasoning","display_name":"deductive reasoning","score":0.8799999952316284},{"id":"https://openalex.org/keywords/abductive-reasoning","display_name":"abductive reasoning","score":0.9980000257492065},{"id":"https://openalex.org/keywords/inductive-reasoning","display_name":"inductive reasoning","score":1.0},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"multi-task learning","score":0.7379999756813049},{"id":"https://openalex.org/keywords/instruction-tuning","display_name":"instruction tuning","score":0.8270000219345093}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7314568161964417},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5780274271965027},{"id":"https://openalex.org/C203702819","wikidata":"https://www.wikidata.org/wiki/Q17146953","display_name":"Logical data model","level":3,"score":0.4374726116657257},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.40501731634140015},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.1841306984424591},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.12077257037162781},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.10101351141929626},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.00836","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.00836","pdf_url":"https://arxiv.org/pdf/2310.00836","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.00836","doi":"https://doi.org/10.48550/arxiv.2310.00836","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.00836","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.00836","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.00836","pdf_url":"https://arxiv.org/pdf/2310.00836","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387323979.pdf","grobid_xml":"https://content.openalex.org/works/W4387323979.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4395482518","https://openalex.org/W4407571837","https://openalex.org/W4407097799","https://openalex.org/W4387687911","https://openalex.org/W7164022507","https://openalex.org/W4404783604","https://openalex.org/W4417300099","https://openalex.org/W4416390673","https://openalex.org/W4416036206","https://openalex.org/W4417167880"],"abstract_inverted_index":{"Logical":[0],"reasoning":[1,67,82,101,174,220],"is":[2],"fundamental":[3],"for":[4,65,111,217,227],"humans":[5],"yet":[6],"presents":[7],"a":[8,59,85,96,115,121,139],"substantial":[9],"challenge":[10],"in":[11,62,80,92,150,191,222,233],"the":[12,36,45,76,89,99,105,166,171,184,212,225],"domain":[13],"of":[14,38,51,78,88,168],"Artificial":[15],"Intelligence.":[16],"Initially,":[17],"researchers":[18],"used":[19],"Knowledge":[20,53],"Representation":[21,54],"and":[22,30,104,133,158,183,214,230],"Reasoning":[23],"(KR)":[24,55],"systems":[25],"that":[26,187],"did":[27],"not":[28],"scale":[29],"required":[31],"non-trivial":[32],"manual":[33],"effort.":[34],"Recently,":[35],"emergence":[37],"large":[39],"language":[40,147],"models":[41],"(LLMs)":[42],"has":[43],"demonstrated":[44],"ability":[46],"to":[47,74,108,164,208],"overcome":[48],"various":[49,179],"limitations":[50],"formal":[52],"systems.":[56],"Consequently,":[57],"there's":[58],"growing":[60],"interest":[61],"using":[63,181],"LLMs":[64,79,110,180,188],"logical":[66,81,100,173,219],"via":[68],"natural":[69],"language.":[70],"This":[71,125],"work":[72],"strives":[73],"understand":[75],"proficiency":[77,221],"by":[83,195],"offering":[84],"brief":[86],"review":[87],"latest":[90],"progress":[91],"this":[93,234],"area;":[94],"with":[95],"focus":[97],"on":[98,211],"datasets,":[102],"tasks,":[103],"methods":[106],"adopted":[107],"utilize":[109],"reasoning.":[112,135,205],"To":[113],"offer":[114],"thorough":[116],"analysis,":[117],"we":[118,141],"have":[119,142],"compiled":[120],"benchmark":[122],"titled":[123],"LogiGLUE.":[124],"includes":[126],"24":[127],"varied":[128],"datasets":[129],"encompassing":[130],"deductive,":[131],"abductive,":[132],"inductive":[134,204],"Utilizing":[136],"LogiGLUE":[137],"as":[138],"foundation,":[140],"trained":[143],"an":[144],"instruction":[145],"fine-tuned":[146],"model,":[148],"resulting":[149],"LogiT5.":[151],"We":[152,176,206],"study":[153],"single-task":[154],"training,":[155,157],"multi-task":[156],"\"chain-of-thought\"":[159],"knowledge":[160],"distillation":[161],"fine-tuning":[162],"technique":[163],"assess":[165,178],"performance":[167],"model":[169],"across":[170],"different":[172],"categories.":[175],"also":[177],"LogiGLUE,":[182],"findings":[185],"indicate":[186],"excel":[189],"most":[190],"abductive":[192],"reasoning,":[193,197],"followed":[194],"deductive":[196],"while":[198],"they":[199],"are":[200],"least":[201],"effective":[202],"at":[203],"aim":[207],"shed":[209],"light":[210],"capabilities":[213],"potential":[215],"pathways":[216],"enhancing":[218],"LLMs,":[223],"paving":[224],"way":[226],"more":[228],"advanced":[229],"nuanced":[231],"developments":[232],"critical":[235],"field.":[236]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":1}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2023-10-04T00:00:00"}