{"id":"https://openalex.org/W7123591232","doi":"https://doi.org/10.1145/3769126.3769128","title":"Identifying Legal Holdings with LLMs: A Systematic Study of Performance, Scale, and Memorization","display_name":"Identifying Legal Holdings with LLMs: A Systematic Study of Performance, Scale, and Memorization","publication_year":2025,"publication_date":"2025-06-16","ids":{"openalex":"https://openalex.org/W7123591232","doi":"https://doi.org/10.1145/3769126.3769128"},"language":null,"primary_location":{"id":"doi:10.1145/3769126.3769128","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769128","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3769126.3769128","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5119753259","display_name":"Chuck Arvin","orcid":null},"institutions":[{"id":"https://openalex.org/I1174212","display_name":"University of Southern California","ror":"https://ror.org/03taz7m60","country_code":"US","type":"education","lineage":["https://openalex.org/I1174212"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Chuck Arvin","raw_affiliation_strings":["Gould School of Law, University of Southern California, Los Angeles, California, USA"],"raw_orcid":"https://orcid.org/0009-0003-9077-2185","affiliations":[{"raw_affiliation_string":"Gould School of Law, University of Southern California, Los Angeles, California, USA","institution_ids":["https://openalex.org/I1174212"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5119753259"],"corresponding_institution_ids":["https://openalex.org/I1174212"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.81522417,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"404","last_page":"408"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6700000166893005,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6700000166893005,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14013","display_name":"Legal Language and Interpretation","score":0.025100000202655792,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13851","display_name":"Law, AI, and Intellectual Property","score":0.020999999716877937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.7063000202178955},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5583000183105469},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.5153999924659729},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.510699987411499},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.3425999879837036},{"id":"https://openalex.org/keywords/analytics","display_name":"Analytics","score":0.33399999141693115}],"concepts":[{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.7063000202178955},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5583000183105469},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.522599995136261},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.5153999924659729},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.510699987411499},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4004000127315521},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38940000534057617},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3695000112056732},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.3617999851703644},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.35670000314712524},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.3425999879837036},{"id":"https://openalex.org/C79158427","wikidata":"https://www.wikidata.org/wiki/Q485396","display_name":"Analytics","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C148324565","wikidata":"https://www.wikidata.org/wiki/Q784221","display_name":"Rote learning","level":4,"score":0.33309999108314514},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.3174999952316284},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2808000147342682},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.27390000224113464},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.26010000705718994}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3769126.3769128","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769128","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3769126.3769128","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769128","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.6629557609558105}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W3186492090","https://openalex.org/W3202026671","https://openalex.org/W4389403907","https://openalex.org/W4402409840","https://openalex.org/W4402683925"],"related_works":[],"abstract_inverted_index":{"As":[0],"large":[1],"language":[2],"models":[3,70],"(LLMs)":[4],"continue":[5],"to":[6,13,30,40,118,166],"advance":[7],"in":[8,124],"capabilities,":[9],"it":[10],"is":[11,163],"essential":[12],"assess":[14,31],"how":[15],"they":[16],"perform":[17],"on":[18,43,60,93],"established":[19],"benchmarks.":[20,197],"In":[21],"this":[22,61,94],"study,":[23],"we":[24,128],"present":[25],"a":[26,45,132],"suite":[27],"of":[28,34,79,120,158,178,191],"experiments":[29,54],"the":[32,89,125,161,173,187],"performance":[33,59,152,162],"modern":[35],"LLMs":[36,179],"(ranging":[37],"from":[38],"3B":[39],"90B+":[41],"parameters)":[42],"CaseHOLD,":[44],"legal":[46,181,193,196],"benchmark":[47],"dataset":[48],"for":[49,180,186],"identifying":[50],"case":[51,143],"holdings.":[52],"Our":[53],"demonstrate":[55,171],"\u201cscaling":[56],"effects\u201d":[57],"-":[58],"task":[62],"improves":[63],"with":[64,67,88,183],"model":[65,103],"size,":[66],"more":[68],"capable":[69],"like":[71],"GPT4o":[72],"and":[73,81,96,130,145,175,189,195],"AmazonNovaPro":[74],"achieving":[75],"macro":[76],"F1":[77,157],"scores":[78,85],"0.744":[80],"0.720":[82],"respectively.":[83],"These":[84,169],"are":[86,115,147],"competitive":[87],"best":[90],"published":[91],"results":[92,114],"dataset,":[95],"do":[97],"not":[98,116,164],"require":[99],"any":[100],"technically":[101],"sophisticated":[102],"training,":[104],"fine-tuning":[105],"or":[106],"few-shot":[107],"prompting.":[108],"To":[109],"ensure":[110],"that":[111,137],"these":[112,154],"strong":[113,151],"due":[117,165],"memorization":[119],"judicial":[121],"opinions":[122],"contained":[123],"training":[126],"data,":[127],"develop":[129],"utilize":[131],"novel":[133],"citation":[134],"anonymization":[135],"test":[136],"preserves":[138],"semantic":[139],"meaning":[140],"while":[141],"ensuring":[142],"names":[144],"citations":[146],"fictitious.":[148],"Models":[149],"maintain":[150],"under":[153],"conditions":[155],"(macro":[156],"0.728),":[159],"suggesting":[160],"rote":[167],"memorization.":[168],"findings":[170],"both":[172],"promise":[174],"current":[176],"limitations":[177],"tasks":[182],"important":[184],"implications":[185],"development":[188],"measurement":[190],"automated":[192],"analytics":[194]},"counts_by_year":[],"updated_date":"2026-01-14T23:44:37.837170","created_date":"2026-01-14T00:00:00"}