{"id":"https://openalex.org/W7140141301","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.228","title":"RiddleBench: A New Generative Reasoning Benchmark for LLMs","display_name":"RiddleBench: A New Generative Reasoning Benchmark for LLMs","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7140141301","doi":"https://doi.org/10.18653/v1/2026.findings-eacl.228"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-eacl.228","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.228","pdf_url":"https://aclanthology.org/2026.findings-eacl.228.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-eacl.228.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Deepon Halder","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Deepon Halder","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Alan Saji","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Alan Saji","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Thanmay Jayakumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Thanmay Jayakumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Anoop Kunchukuttan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Anoop Kunchukuttan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ratish Puduppully","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ratish Puduppully","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Raj Dabre","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raj Dabre","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4363","last_page":"4372"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.19009999930858612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.19009999930858612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.06369999796152115,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.05869999900460243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5270000100135803},{"id":"https://openalex.org/keywords/generative-grammar","display_name":"Generative grammar","score":0.429500013589859},{"id":"https://openalex.org/keywords/case-based-reasoning","display_name":"Case-based reasoning","score":0.32910001277923584},{"id":"https://openalex.org/keywords/generative-model","display_name":"Generative model","score":0.2881999909877777},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.2872999906539917}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5805000066757202},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5601999759674072},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5270000100135803},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.429500013589859},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3264999985694885},{"id":"https://openalex.org/C167966045","wikidata":"https://www.wikidata.org/wiki/Q5532625","display_name":"Generative model","level":3,"score":0.2881999909877777},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2872999906539917},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.260699987411499},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.2468000054359436}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-eacl.228","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.228","pdf_url":"https://aclanthology.org/2026.findings-eacl.228.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-eacl.228","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-eacl.228","pdf_url":"https://aclanthology.org/2026.findings-eacl.228.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7140141301.pdf","grobid_xml":"https://content.openalex.org/works/W7140141301.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"demonstrated":[5],"impressive":[6],"performance":[7,150],"on":[8,87],"many":[9],"established":[10],"reasoning":[11,32,80,132,146],"benchmarks.However,":[12],"these":[13,78,167],"benchmarks":[14],"primarily":[15],"evaluate":[16],"structured":[17],"skills":[18,41],"like":[19,96],"quantitative":[20],"problem-solving,":[21],"leaving":[22],"a":[23,36,65,141,163,172],"critical":[24,168],"gap":[25],"in":[26,72],"assessing":[27],"the":[28,177],"more":[29,180],"flexible,":[30],"multifaceted":[31],"abilities":[33],"that":[34],"are":[35],"cornerstone":[37],"of":[38,68,84,109,119,133,179],"human":[39],"intelligence.These":[40],"require":[42],"synthesizing":[43],"logical":[44],"deduction":[45],"with":[46,149,155],"spatial":[47],"awareness":[48],"and":[49,101,115,136,170,182],"constraint":[50],"satisfaction,":[51],"which":[52],"current":[53],"evaluations":[54],"do":[55],"not":[56],"adequately":[57],"measure.To":[58],"address":[59],"this":[60],"gap,":[61],"we":[62],"introduce":[63],"RiddleBench,":[64],"new":[66],"benchmark":[67],"1,737":[69],"challenging":[70],"puzzles":[71],"English":[73],"designed":[74],"specifically":[75],"to":[76,140],"probe":[77],"core":[79],"capabilities.Our":[81],"comprehensive":[82],"evaluation":[83],"state-of-the-art":[85],"models":[86,95,121],"RiddleBench":[88],"reveals":[89,123],"fundamental":[90],"weaknesses;":[91],"even":[92],"top-tier":[93],"proprietary":[94],"Gemini":[97],"2.5":[98],"Pro,":[99],"o3,":[100],"Claude":[102],"4":[103],"Sonnet":[104],"achieve":[105],"overall":[106],"accuracy":[107],"scores":[108],"just":[110],"above":[111],"60%":[112],"(60.30%,":[113],"63.37%,":[114],"63.16%,":[116],"respectively).Our":[117],"analysis":[118],"specific":[120],"further":[122],"deep":[124],"failures,":[125],"including":[126],"\"hallucination":[127],"cascades\"":[128],"(uncritically":[129],"accepting":[130],"flawed":[131],"other/evaluated":[134],"LLMs)":[135],"poor":[137],"self-correction":[138],"due":[139],"strong":[142],"self-confirmation":[143],"bias.Furthermore,":[144],"their":[145],"proves":[147],"fragile,":[148],"degrading":[151],"significantly":[152],"when":[153],"faced":[154],"reordered":[156],"constraints":[157],"or":[158],"irrelevant":[159],"information.RiddleBench":[160],"serves":[161],"as":[162],"diagnostic":[164],"tool":[165],"for":[166,175],"issues":[169],"provides":[171],"valuable":[173],"resource":[174],"guiding":[176],"development":[178],"robust":[181],"reliable":[183],"LLMs.":[184]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-12-10T00:00:00"}