{"id":"https://openalex.org/W4412887967","doi":"https://doi.org/10.18653/v1/2025.findings-acl.905","title":"Are Your LLMs Capable of Stable Reasoning?","display_name":"Are Your LLMs Capable of Stable Reasoning?","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4412887967","doi":"https://doi.org/10.18653/v1/2025.findings-acl.905"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2025.findings-acl.905","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.905","pdf_url":"https://aclanthology.org/2025.findings-acl.905.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2025.findings-acl.905.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051994745","display_name":"Junnan Liu","orcid":"https://orcid.org/0009-0005-7629-947X"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junnan Liu","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100411939","display_name":"Hongwei Liu","orcid":"https://orcid.org/0000-0001-6471-131X"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongwei Liu","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077857247","display_name":"Linchen Xiao","orcid":"https://orcid.org/0000-0002-8913-4771"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linchen Xiao","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100441368","display_name":"Ziyi Wang","orcid":"https://orcid.org/0000-0002-1334-7287"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziyi Wang","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050684712","display_name":"Kuikun Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kuikun Liu","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109932574","display_name":"Songyang Gao","orcid":"https://orcid.org/0000-0001-5485-8019"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Songyang Gao","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100619686","display_name":"Wenwei Zhang","orcid":"https://orcid.org/0000-0002-2745-9773"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wenwei Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100747793","display_name":"Songyang Zhang","orcid":"https://orcid.org/0000-0002-9297-0201"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Songyang Zhang","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100437934","display_name":"Kai Chen","orcid":"https://orcid.org/0000-0002-6108-8283"},"institutions":[{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kai Chen","raw_affiliation_strings":["Shanghai Artificial Intelligence Laboratory"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Artificial Intelligence Laboratory","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"17594","last_page":"17632"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9693999886512756,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9693999886512756,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4778289794921875}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4778289794921875}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2025.findings-acl.905","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.905","pdf_url":"https://aclanthology.org/2025.findings-acl.905.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2025.findings-acl.905","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2025.findings-acl.905","pdf_url":"https://aclanthology.org/2025.findings-acl.905.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2025","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412887967.pdf","grobid_xml":"https://content.openalex.org/works/W4412887967.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"The":[0],"rapid":[1],"advancement":[2],"of":[3,42,125],"large":[4,100],"language":[5,101],"models":[6,102],"(LLMs)":[7],"has":[8],"shown":[9],"remarkable":[10],"progress":[11],"in":[12,46,96],"complex":[13,47],"reasoning":[14,48,123],"tasks.However,":[15],"a":[16,62,116],"significant":[17,117],"disparity":[18],"exists":[19],"between":[20],"benchmark":[21],"performances":[22],"and":[23,34,53,81,89,111],"real-world":[24],"applications.We":[25],"attribute":[26],"this":[27,57],"gap":[28],"primarily":[29],"to":[30,103,119],"current":[31],"evaluation":[32,64,133],"protocols":[33],"metrics,":[35],"which":[36],"inadequately":[37],"capture":[38],"the":[39,77,121,128],"full":[40],"spectrum":[41],"LLM":[43],"capabilities,":[44],"especially":[45],"tasks":[49],"where":[50],"both":[51,76],"accuracy":[52],"consistency":[54],"are":[55],"essential.In":[56],"paper,":[58],"we":[59,93],"introduce":[60],"G-Pass@k,":[61],"novel":[63],"metric":[65],"that":[66],"continuously":[67],"assesses":[68],"model":[69],"performance":[70,79],"across":[71],"multiple":[72],"sampling":[73],"attempts,":[74],"quantifying":[75],"model's":[78],"potential":[80,109],"its":[82],"stability.Through":[83],"extensive":[84],"experiments":[85],"on":[86],"various":[87],"public":[88],"newly":[90],"constructed":[91],"benchmarks,":[92],"employ":[94],"G-Pass@k":[95],"conjunction":[97],"with":[98],"state-of-the-art":[99],"provide":[104],"comprehensive":[105],"insights":[106],"into":[107],"their":[108],"capabilities":[110],"operational":[112],"consistency.Our":[113],"findings":[114],"reveal":[115],"opportunity":[118],"enhance":[120],"realistic":[122],"abilities":[124],"LLMs,":[126],"underscoring":[127],"necessity":[129],"for":[130],"more":[131],"robust":[132],"metrics.":[134]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}