{"id":"https://openalex.org/W4408395322","doi":"https://doi.org/10.1145/3709025.3712219","title":"A Reasoning-Focused Legal Retrieval Benchmark","display_name":"A Reasoning-Focused Legal Retrieval Benchmark","publication_year":2025,"publication_date":"2025-03-13","ids":{"openalex":"https://openalex.org/W4408395322","doi":"https://doi.org/10.1145/3709025.3712219"},"language":"en","primary_location":{"id":"doi:10.1145/3709025.3712219","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3709025.3712219","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3709025.3712219?download=true","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Symposium on Computer Science and Law on ZZZ","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3709025.3712219?download=true","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5052391231","display_name":"Lucia Zheng","orcid":"https://orcid.org/0000-0002-8602-0007"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Lucia Zheng","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-8602-0007","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068178240","display_name":"Neel Guha","orcid":"https://orcid.org/0009-0003-5120-1726"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Neel Guha","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0009-0003-5120-1726","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116304948","display_name":"Javokhir Arifov","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Javokhir Arifov","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0009-0009-2442-6042","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sarah Zhang","orcid":"https://orcid.org/0009-0009-0265-5071"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sarah Zhang","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0009-0009-0265-5071","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083430717","display_name":"Michal Skreta","orcid":"https://orcid.org/0000-0002-9999-2737"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Michal Skreta","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-9999-2737","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046006076","display_name":"Christopher D. Manning","orcid":"https://orcid.org/0000-0001-6155-649X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christopher D. Manning","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0000-0001-6155-649X","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049073875","display_name":"Peter Henderson","orcid":"https://orcid.org/0000-0003-3938-0541"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Peter Henderson","raw_affiliation_strings":["Princeton University, Princeton, New Jersey, USA"],"raw_orcid":"https://orcid.org/0000-0003-3938-0541","affiliations":[{"raw_affiliation_string":"Princeton University, Princeton, New Jersey, USA","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5058408154","display_name":"Daniel E. Ho","orcid":"https://orcid.org/0000-0002-2195-5469"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel E. Ho","raw_affiliation_strings":["Stanford University, Stanford, California, USA"],"raw_orcid":"https://orcid.org/0000-0002-2195-5469","affiliations":[{"raw_affiliation_string":"Stanford University, Stanford, California, USA","institution_ids":["https://openalex.org/I97018004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5052391231"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":45.5865,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.99701957,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"169","last_page":"193"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12755","display_name":"Legal Education and Practice Innovations","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11762","display_name":"Law, Economics, and Judicial Systems","score":0.982200026512146,"subfield":{"id":"https://openalex.org/subfields/2002","display_name":"Economics and Econometrics"},"field":{"id":"https://openalex.org/fields/20","display_name":"Economics, Econometrics and Finance"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7359229326248169},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6975098848342896},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5382250547409058},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.43130558729171753},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3246731162071228},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07116097211837769},{"id":"https://openalex.org/keywords/cartography","display_name":"Cartography","score":0.05028840899467468}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7359229326248169},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6975098848342896},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5382250547409058},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43130558729171753},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3246731162071228},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07116097211837769},{"id":"https://openalex.org/C58640448","wikidata":"https://www.wikidata.org/wiki/Q42515","display_name":"Cartography","level":1,"score":0.05028840899467468}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3709025.3712219","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3709025.3712219","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3709025.3712219?download=true","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Symposium on Computer Science and Law on ZZZ","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2505.03970","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.03970","pdf_url":"https://arxiv.org/pdf/2505.03970","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3709025.3712219","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3709025.3712219","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3709025.3712219?download=true","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Symposium on Computer Science and Law on ZZZ","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408395322.pdf","grobid_xml":"https://content.openalex.org/works/W4408395322.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2043909051","https://openalex.org/W2044405664","https://openalex.org/W2798658104","https://openalex.org/W2910577570","https://openalex.org/W2963339397","https://openalex.org/W2990138404","https://openalex.org/W2998733856","https://openalex.org/W3121809023","https://openalex.org/W3186492090","https://openalex.org/W3195296860","https://openalex.org/W4393161188","https://openalex.org/W4399036020"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W2745001401","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W4246352526"],"abstract_inverted_index":{"As":[0],"the":[1,6,36,43,52,99,105],"legal":[2,15,17,47,56,60,69,84,95,115],"community":[3],"increasingly":[4],"examines":[5],"use":[7],"of":[8,38,45,54,101,107],"large":[9],"language":[10],"models":[11],"(LLMs)":[12],"for":[13],"various":[14],"applications,":[16],"AI":[18],"developers":[19],"have":[20],"turned":[21],"to":[22,27,35,82],"retrieval-augmented":[23],"LLMs":[24],"(\"RAG\"":[25],"systems)":[26],"improve":[28],"system":[29],"performance":[30,106],"and":[31,58,75,87,104],"robustness.":[32],"An":[33],"obstacle":[34],"development":[37],"specialized":[39],"RAG":[40,48,70,116],"systems":[41],"is":[42],"lack":[44],"realistic":[46],"benchmarks":[49,103],"which":[50,93],"capture":[51],"complexity":[53],"both":[55],"retrieval":[57],"downstream":[59],"question-answering.":[61],"To":[62],"address":[63],"this,":[64],"we":[65],"introduce":[66],"two":[67],"novel":[68],"benchmarks:":[71],"Bar":[72],"Exam":[73],"QA":[74],"Housing":[76],"Statute":[77],"QA.":[78],"Our":[79,111],"tasks":[80],"correspond":[81],"real-world":[83],"research":[85],"tasks,":[86],"were":[88],"produced":[89],"through":[90],"annotation":[91],"processes":[92],"resemble":[94],"research.":[96,124],"We":[97],"describe":[98],"construction":[100],"these":[102],"existing":[108],"retriever":[109],"pipelines.":[110],"results":[112],"suggest":[113],"that":[114],"remains":[117],"a":[118],"challenging":[119],"application,":[120],"thus":[121],"motivating":[122],"future":[123]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}