{"id":"https://openalex.org/W4416988068","doi":"https://doi.org/10.3233/faia251592","title":"Do LLMs Truly \u201cUnderstand\u201d when a Precedent Is Overruled?","display_name":"Do LLMs Truly \u201cUnderstand\u201d when a Precedent Is Overruled?","publication_year":2025,"publication_date":"2025-12-02","ids":{"openalex":"https://openalex.org/W4416988068","doi":"https://doi.org/10.3233/faia251592"},"language":null,"primary_location":{"id":"doi:10.3233/faia251592","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251592","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"type":"book-chapter","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.3233/faia251592","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015380093","display_name":"L. Zhang","orcid":"https://orcid.org/0000-0003-0375-1793"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Li Zhang","raw_affiliation_strings":["University of Pittsburgh"],"raw_orcid":"https://orcid.org/0000-0003-0375-1793","affiliations":[{"raw_affiliation_string":"University of Pittsburgh","institution_ids":["https://openalex.org/I170201317"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077762937","display_name":"Jarom\u00edr \u0160avelka","orcid":"https://orcid.org/0000-0002-3674-5456"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jaromir Savelka","raw_affiliation_strings":["Carnegie Mellon University"],"raw_orcid":"https://orcid.org/0000-0002-3674-5456","affiliations":[{"raw_affiliation_string":"Carnegie Mellon University","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101664770","display_name":"Kevin D. Ashley","orcid":"https://orcid.org/0000-0002-5535-0759"},"institutions":[{"id":"https://openalex.org/I170201317","display_name":"University of Pittsburgh","ror":"https://ror.org/01an3r305","country_code":"US","type":"education","lineage":["https://openalex.org/I170201317"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Ashley","raw_affiliation_strings":["University of Pittsburgh"],"raw_orcid":"https://orcid.org/0000-0002-5535-0759","affiliations":[{"raw_affiliation_string":"University of Pittsburgh","institution_ids":["https://openalex.org/I170201317"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5015380093"],"corresponding_institution_ids":["https://openalex.org/I170201317"],"apc_list":null,"apc_paid":null,"fwci":21.9819,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.99284498,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6148999929428101,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6148999929428101,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14013","display_name":"Legal Language and Interpretation","score":0.07540000230073929,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.05939999967813492,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/heuristics","display_name":"Heuristics","score":0.6320000290870667},{"id":"https://openalex.org/keywords/doctrine","display_name":"Doctrine","score":0.5852000117301941},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.560699999332428},{"id":"https://openalex.org/keywords/supreme-court","display_name":"Supreme court","score":0.3684000074863434},{"id":"https://openalex.org/keywords/legal-doctrine","display_name":"Legal doctrine","score":0.3544999957084656},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.3483000099658966},{"id":"https://openalex.org/keywords/deductive-reasoning","display_name":"Deductive reasoning","score":0.34299999475479126},{"id":"https://openalex.org/keywords/common-law","display_name":"Common law","score":0.33230000734329224}],"concepts":[{"id":"https://openalex.org/C127705205","wikidata":"https://www.wikidata.org/wiki/Q5748245","display_name":"Heuristics","level":2,"score":0.6320000290870667},{"id":"https://openalex.org/C2776211767","wikidata":"https://www.wikidata.org/wiki/Q117850","display_name":"Doctrine","level":2,"score":0.5852000117301941},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.560699999332428},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5016999840736389},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.43299999833106995},{"id":"https://openalex.org/C2778272461","wikidata":"https://www.wikidata.org/wiki/Q190752","display_name":"Supreme court","level":2,"score":0.3684000074863434},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.35589998960494995},{"id":"https://openalex.org/C2778890055","wikidata":"https://www.wikidata.org/wiki/Q1192543","display_name":"Legal doctrine","level":2,"score":0.3544999957084656},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3483000099658966},{"id":"https://openalex.org/C97364631","wikidata":"https://www.wikidata.org/wiki/Q484284","display_name":"Deductive reasoning","level":2,"score":0.34299999475479126},{"id":"https://openalex.org/C170706310","wikidata":"https://www.wikidata.org/wiki/Q30216","display_name":"Common law","level":2,"score":0.33230000734329224},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.3280999958515167},{"id":"https://openalex.org/C100505606","wikidata":"https://www.wikidata.org/wiki/Q6302997","display_name":"Judicial opinion","level":2,"score":0.3237999975681305},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.31779998540878296},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.30230000615119934},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C2780938662","wikidata":"https://www.wikidata.org/wiki/Q973710","display_name":"Tying","level":2,"score":0.2953999936580658},{"id":"https://openalex.org/C31395832","wikidata":"https://www.wikidata.org/wiki/Q1318674","display_name":"Testbed","level":2,"score":0.29249998927116394},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2872999906539917},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.25769999623298645},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.2574000060558319},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.2572999894618988},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.25290000438690186},{"id":"https://openalex.org/C200635333","wikidata":"https://www.wikidata.org/wiki/Q158970","display_name":"Tort","level":3,"score":0.2513999938964844},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.25119999051094055}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3233/faia251592","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251592","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},{"id":"pmh:oai:arXiv.org:2510.20941","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.20941","pdf_url":"https://arxiv.org/pdf/2510.20941","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.3233/faia251592","is_oa":true,"landing_page_url":"https://doi.org/10.3233/faia251592","pdf_url":null,"source":{"id":"https://openalex.org/S4210201731","display_name":"Frontiers in artificial intelligence and applications","issn_l":"0922-6389","issn":["0922-6389","1879-8314"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Frontiers in Artificial Intelligence and Applications","raw_type":"book-chapter"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,126,148,165],"(LLMs)":[3],"with":[4],"extended":[5],"context":[6],"windows":[7],"show":[8,127],"promise":[9],"for":[10,80],"complex":[11,171],"legal":[12,21,82,88,157,207],"reasoning":[13,146,162,208],"tasks,":[14],"yet":[15],"their":[16,142],"ability":[17],"to":[18,52,64,134],"understand":[19],"long":[20],"documents":[22],"remains":[23,34],"insufficiently":[24],"evaluated.":[25],"Developing":[26],"long-context":[27,194],"benchmarks":[28],"that":[29,50,84,187,199],"capture":[30],"realistic,":[31],"high-stakes":[32],"tasks":[33,49,173],"a":[35,75,109,185],"significant":[36],"challenge":[37],"in":[38,70,141,170,179,192],"the":[39,54,125,189,201],"field,":[40],"as":[41],"most":[42],"existing":[43],"evaluations":[44],"rely":[45,149],"on":[46,99,130,150],"simplified":[47],"synthetic":[48],"fail":[51],"represent":[53],"complexity":[55,202],"of":[56,96,111,205],"real-world":[57],"document":[58],"understanding.":[59],"Overruling":[60],"relationships":[61,102,169],"are":[62],"foundational":[63],"common-law":[65],"doctrine":[66],"and":[67,77,159,203],"commonly":[68],"found":[69],"judicial":[71],"opinions.":[72],"They":[73],"provide":[74],"focused":[76],"important":[78],"testbed":[79],"long-document":[81],"understanding":[83],"closely":[85],"resembles":[86],"what":[87],"professionals":[89],"actually":[90],"do.":[91],"We":[92],"present":[93],"an":[94,197],"assessment":[95],"state-of-the-art":[97],"LLMs":[98],"identifying":[100],"overruling":[101],"from":[103],"U.S.":[104],"Supreme":[105],"Court":[106],"cases":[107,132],"using":[108],"dataset":[110,212],"236":[112],"case":[113],"pairs.":[114],"Our":[115,182],"evaluation":[116],"reveals":[117],"three":[118],"critical":[119,190],"limitations:":[120],"(1)":[121],"era":[122],"sensitivity":[123],"\u2013":[124,147,164],"degraded":[128],"performance":[129],"historical":[131],"compared":[133],"modern":[135],"ones,":[136],"revealing":[137],"fundamental":[138],"temporal":[139,177],"bias":[140],"training;":[143],"(2)":[144],"shallow":[145,151],"logical":[152],"heuristics":[153],"rather":[154],"than":[155],"deep":[156],"comprehension;":[158],"(3)":[160],"context-dependent":[161],"failures":[163],"produce":[166],"temporally":[167],"impossible":[168],"open-ended":[172],"despite":[174],"maintaining":[175],"basic":[176],"awareness":[178],"simple":[180],"contexts.":[181],"work":[183],"contributes":[184],"benchmark":[186],"addresses":[188],"gap":[191],"realistic":[193],"evaluation,":[195],"providing":[196],"environment":[198],"mirrors":[200],"stakes":[204],"actual":[206],"tasks.":[209],"The":[210],"full":[211],"can":[213],"be":[214],"accessed":[215],"at":[216],"https://github.com/lizhang-AIandLaw/Do-LLMs-Truly-Understand-When-a-Precedent-Is-Overruled":[217]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-05-17T08:19:37.847499","created_date":"2025-12-04T00:00:00"}