{"id":"https://openalex.org/W7123671523","doi":"https://doi.org/10.1145/3769126.3769245","title":"LLMs Provide Unstable Answers to Legal Questions","display_name":"LLMs Provide Unstable Answers to Legal Questions","publication_year":2025,"publication_date":"2025-06-16","ids":{"openalex":"https://openalex.org/W7123671523","doi":"https://doi.org/10.1145/3769126.3769245"},"language":null,"primary_location":{"id":"doi:10.1145/3769126.3769245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769245","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3769126.3769245","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5077212858","display_name":"Andrew Blair-Stanek","orcid":"https://orcid.org/0000-0001-8547-9417"},"institutions":[{"id":"https://openalex.org/I126744593","display_name":"University of Maryland, Baltimore","ror":"https://ror.org/04rq5mt64","country_code":"US","type":"education","lineage":["https://openalex.org/I126744593"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew Blair-Stanek","raw_affiliation_strings":["Carey School of Law, University of Maryland, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0001-8547-9417","affiliations":[{"raw_affiliation_string":"Carey School of Law, University of Maryland, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I126744593"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075825791","display_name":"Benjamin Van Durme","orcid":"https://orcid.org/0000-0003-4328-4288"},"institutions":[{"id":"https://openalex.org/I145311948","display_name":"Johns Hopkins University","ror":"https://ror.org/00za53h95","country_code":"US","type":"education","lineage":["https://openalex.org/I145311948"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Benjamin Van Durme","raw_affiliation_strings":["Computer Science, Johns Hopkins University, Baltimore, Maryland, USA"],"raw_orcid":"https://orcid.org/0000-0003-4328-4288","affiliations":[{"raw_affiliation_string":"Computer Science, Johns Hopkins University, Baltimore, Maryland, USA","institution_ids":["https://openalex.org/I145311948"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"425","last_page":"429"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6636999845504761,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.6636999845504761,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.0494999997317791,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.03750000149011612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/legal-research","display_name":"Legal research","score":0.321399986743927},{"id":"https://openalex.org/keywords/legal-ethics","display_name":"Legal ethics","score":0.31520000100135803},{"id":"https://openalex.org/keywords/legal-case","display_name":"Legal case","score":0.2572000026702881},{"id":"https://openalex.org/keywords/legal-action","display_name":"Legal action","score":0.2371000051498413}],"concepts":[{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.5623999834060669},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.4984999895095825},{"id":"https://openalex.org/C190253527","wikidata":"https://www.wikidata.org/wiki/Q295354","display_name":"Law and economics","level":1,"score":0.382099986076355},{"id":"https://openalex.org/C522695570","wikidata":"https://www.wikidata.org/wiki/Q6517578","display_name":"Legal research","level":2,"score":0.321399986743927},{"id":"https://openalex.org/C503263630","wikidata":"https://www.wikidata.org/wiki/Q3655952","display_name":"Legal ethics","level":2,"score":0.31520000100135803},{"id":"https://openalex.org/C2778049185","wikidata":"https://www.wikidata.org/wiki/Q2334719","display_name":"Legal case","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.24050000309944153},{"id":"https://openalex.org/C2993819037","wikidata":"https://www.wikidata.org/wiki/Q836925","display_name":"Legal action","level":2,"score":0.2371000051498413},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.22419999539852142},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.22030000388622284}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3769126.3769245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769245","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3769126.3769245","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3769126.3769245","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twentieth International Conference on Artificial Intelligence and Law","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7169655561447144,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G1494501847","display_name":"Computational Statutory Reasoning","funder_award_id":"2204926","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":5,"referenced_works":["https://openalex.org/W3122184684","https://openalex.org/W4281557260","https://openalex.org/W4402385990","https://openalex.org/W4403499178","https://openalex.org/W4404781852"],"related_works":[],"abstract_inverted_index":{"An":[0],"LLM":[1],"is":[2],"\u2018\u2018stable\u2019\u2019":[3],"if":[4],"it":[5],"reaches":[6],"the":[7,12,71,80,98,108],"same":[8,82],"conclusion":[9],"when":[10,28,36],"asked":[11],"identical":[13],"question":[14,72],"multiple":[15],"times.":[16],"We":[17,47],"find":[18],"leading":[19],"LLMs":[20,87],"like":[21],"gpt-4o,":[22],"claude-3.5,":[23],"and":[24,49,70,117],"gemini-1.5":[25],"are":[26],"unstable":[27],"providing":[29],"answers":[30],"to":[31,45],"hard":[32],"legal":[33,56,68,112,115],"questions,":[34],"even":[35],"made":[37],"as":[38,40],"deterministic":[39],"possible":[41],"by":[42],"setting":[43],"temperature":[44],"0.":[46],"curate":[48],"release":[50],"a":[51],"novel":[52],"dataset":[53],"of":[54,73,111],"500":[55],"questions":[57],"distilled":[58],"from":[59],"real":[60],"cases,":[61],"involving":[62],"two":[63],"parties,":[64],"with":[65],"facts,":[66],"competing":[67],"arguments,":[69],"which":[74],"party":[75,91,100],"should":[76,92,101],"prevail.":[77],"When":[78],"provided":[79],"exact":[81],"question,":[83],"we":[84],"observe":[85],"that":[86],"sometimes":[88],"say":[89,97],"one":[90],"win,":[93],"while":[94],"other":[95,99],"times":[96],"win.":[102],"This":[103],"instability":[104],"has":[105],"implications":[106],"for":[107],"increasing":[109],"numbers":[110],"AI":[113],"products,":[114],"processes,":[116],"lawyers":[118],"relying":[119],"on":[120],"these":[121],"LLMs.":[122]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2026-01-14T00:00:00"}