{"id":"https://openalex.org/W7164933763","doi":"https://doi.org/10.48550/arxiv.2606.16118","title":"Know Your Limits : On the Faithfulness of LLMs as Solvers and Autoformalizers in Legal Reasoning","display_name":"Know Your Limits : On the Faithfulness of LLMs as Solvers and Autoformalizers in Legal Reasoning","publication_year":2026,"publication_date":"2026-06-15","ids":{"openalex":"https://openalex.org/W7164933763","doi":"https://doi.org/10.48550/arxiv.2606.16118"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.16118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.16118","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120717706","display_name":"Olivia Peiyu Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Olivia Peiyu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115506644","display_name":"Sanna Wong-Toropainen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wong-Toropainen, Sanna","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013311087","display_name":"Daneshvar Amrollahi","orcid":"https://orcid.org/0000-0003-0954-7881"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Amrollahi, Daneshvar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120752546","display_name":"Ryan Bai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bai, Ryan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102014833","display_name":"T.K. Bansal","orcid":"https://orcid.org/0009-0008-3024-2576"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bansal, Tashvi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138709409","display_name":"Arush Garg","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Garg, Arush","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5138704540","display_name":"Leilani H. Gilpin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gilpin, Leilani H.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.15230000019073486,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.15230000019073486,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11620000004768372,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.11100000143051147,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/logical-consequence","display_name":"Logical consequence","score":0.5580000281333923},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5496000051498413},{"id":"https://openalex.org/keywords/automated-reasoning","display_name":"Automated reasoning","score":0.5228999853134155},{"id":"https://openalex.org/keywords/interpretation","display_name":"Interpretation (philosophy)","score":0.4494999945163727},{"id":"https://openalex.org/keywords/deductive-reasoning","display_name":"Deductive reasoning","score":0.4156000018119812},{"id":"https://openalex.org/keywords/logical-reasoning","display_name":"Logical reasoning","score":0.39730000495910645},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.39570000767707825},{"id":"https://openalex.org/keywords/abductive-reasoning","display_name":"Abductive reasoning","score":0.3741999864578247},{"id":"https://openalex.org/keywords/theory","display_name":"Theory","score":0.36340001225471497}],"concepts":[{"id":"https://openalex.org/C134752490","wikidata":"https://www.wikidata.org/wiki/Q374182","display_name":"Logical consequence","level":2,"score":0.5580000281333923},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5496000051498413},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.5228999853134155},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4903999865055084},{"id":"https://openalex.org/C527412718","wikidata":"https://www.wikidata.org/wiki/Q855395","display_name":"Interpretation (philosophy)","level":2,"score":0.4494999945163727},{"id":"https://openalex.org/C97364631","wikidata":"https://www.wikidata.org/wiki/Q484284","display_name":"Deductive reasoning","level":2,"score":0.4156000018119812},{"id":"https://openalex.org/C43971567","wikidata":"https://www.wikidata.org/wiki/Q3142865","display_name":"Logical reasoning","level":2,"score":0.39730000495910645},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C166088908","wikidata":"https://www.wikidata.org/wiki/Q308495","display_name":"Abductive reasoning","level":2,"score":0.3741999864578247},{"id":"https://openalex.org/C144701195","wikidata":"https://www.wikidata.org/wiki/Q10859910","display_name":"Theory","level":2,"score":0.36340001225471497},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.35179999470710754},{"id":"https://openalex.org/C44725695","wikidata":"https://www.wikidata.org/wiki/Q288156","display_name":"Normative","level":2,"score":0.34850001335144043},{"id":"https://openalex.org/C2776036281","wikidata":"https://www.wikidata.org/wiki/Q48769818","display_name":"Constraint (computer-aided design)","level":2,"score":0.3452000021934509},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3425999879837036},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.33959999680519104},{"id":"https://openalex.org/C75606506","wikidata":"https://www.wikidata.org/wiki/Q1049183","display_name":"Formal methods","level":2,"score":0.3305000066757202},{"id":"https://openalex.org/C159032336","wikidata":"https://www.wikidata.org/wiki/Q2488768","display_name":"Non-monotonic logic","level":2,"score":0.30730000138282776},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.30169999599456787},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.298799991607666},{"id":"https://openalex.org/C89288958","wikidata":"https://www.wikidata.org/wiki/Q7301504","display_name":"Reasoning system","level":2,"score":0.2865000069141388},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.27410000562667847},{"id":"https://openalex.org/C183521366","wikidata":"https://www.wikidata.org/wiki/Q7256422","display_name":"Psychology of reasoning","level":4,"score":0.27090001106262207},{"id":"https://openalex.org/C102315432","wikidata":"https://www.wikidata.org/wiki/Q649732","display_name":"Formal system","level":2,"score":0.2680000066757202},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.26759999990463257},{"id":"https://openalex.org/C2776325391","wikidata":"https://www.wikidata.org/wiki/Q6917865","display_name":"Motivated reasoning","level":3,"score":0.26489999890327454},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.2565000057220459},{"id":"https://openalex.org/C74992188","wikidata":"https://www.wikidata.org/wiki/Q239179","display_name":"Non-classical logic","level":3,"score":0.25600001215934753},{"id":"https://openalex.org/C3746660","wikidata":"https://www.wikidata.org/wiki/Q1068763","display_name":"Rule of inference","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.16118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.16118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.16118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8212417960166931,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"achieve":[4],"strong":[5],"performance":[6],"on":[7,49],"reasoning":[8,184],"tasks,":[9],"but":[10,140],"whether":[11],"this":[12,24,108],"reflects":[13],"faithful":[14,113],"logical":[15,149,201],"inference":[16],"or":[17],"heuristic":[18],"approximation":[19],"remains":[20],"unclear.":[21],"We":[22,115],"study":[23],"question":[25],"in":[26,152],"legal":[27,68],"entailment":[28],"by":[29],"comparing":[30],"three":[31,117],"paradigms,":[32],"including":[33],"pure":[34],"LLM":[35],"classification,":[36],"LLM-based":[37,97,182],"Formal":[38,42,98],"Reasoning,":[39],"and":[40,63,70,155,200],"solver-based":[41],"Reasoning":[43,99],"using":[44],"the":[45,101,130,179],"Z3":[46,163],"SMT":[47],"solver,":[48],"a":[50,61,75,186,194],"re-annotated":[51],"subset":[52],"of":[53,78,181],"ContractNLI":[54],"across":[55,172],"five":[56],"LLMs.":[57],"Our":[58],"re-annotation":[59],"reveals":[60],"systematic":[62],"measurable":[64],"gap":[65,196],"between":[66,197],"pragmatic":[67],"interpretation":[69],"strict":[71],"formal":[72,92,132,153,183],"entailment,":[73],"where":[74,123,146,159],"substantial":[76],"proportion":[77],"legally":[79],"sound":[80],"inferences":[81],"are":[82,141],"not":[83,111],"formally":[84],"grounded":[85,139],"without":[86,128],"additional":[87],"unstated":[88],"assumptions.":[89],"While":[90],"introducing":[91],"structure":[93],"improves":[94],"accuracy,":[95],"with":[96],"achieving":[100],"highest":[102],"benchmark":[103,198],"performance,":[104],"we":[105],"show":[106],"that":[107,136],"gain":[109],"does":[110],"imply":[112],"reasoning.":[114],"identify":[116],"recurring":[118],"failure":[119],"modes:":[120],"scope":[121,169],"laundering,":[122],"LLMs":[124,147,160],"report":[125],"solver-inconsistent":[126],"classifications":[127],"executing":[129],"underlying":[131],"reasoning,":[133],"producing":[134],"conclusions":[135],"appear":[137],"logically":[138],"not;":[142],"implicit":[143],"constraint":[144],"blindness,":[145],"overlook":[148],"constraints":[150],"present":[151],"representations;":[154],"program":[156],"synthesis":[157],"failures,":[158],"generate":[161],"incorrect":[162],"code":[164],"despite":[165],"structured":[166],"prompting.":[167],"Critically,":[168],"laundering":[170],"persists":[171],"all":[173],"models,":[174],"raising":[175],"serious":[176],"concerns":[177],"about":[178],"faithfulness":[180],"as":[185],"proxy":[187],"for":[188],"symbolic":[189],"execution.":[190],"These":[191],"results":[192],"reveal":[193],"fundamental":[195],"accuracy":[199],"faithfulness.":[202]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-17T00:00:00"}