{"id":"https://openalex.org/W4412377903","doi":"https://doi.org/10.1145/3726302.3730340","title":"LexRAG: Benchmarking Retrieval-Augmented Generation in Multi-Turn Legal Consultation Conversation","display_name":"LexRAG: Benchmarking Retrieval-Augmented Generation in Multi-Turn Legal Consultation Conversation","publication_year":2025,"publication_date":"2025-07-13","ids":{"openalex":"https://openalex.org/W4412377903","doi":"https://doi.org/10.1145/3726302.3730340"},"language":"en","primary_location":{"id":"doi:10.1145/3726302.3730340","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730340","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3726302.3730340","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101638981","display_name":"Haitao Li","orcid":"https://orcid.org/0009-0006-8766-8610"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haitao Li","raw_affiliation_strings":["DCST, Tsinghua University, Beijing, China and Quan Cheng Laboratory, Shandong, China"],"raw_orcid":"https://orcid.org/0009-0006-8766-8610","affiliations":[{"raw_affiliation_string":"DCST, Tsinghua University, Beijing, China and Quan Cheng Laboratory, Shandong, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yifan Chen","orcid":"https://orcid.org/0009-0007-5364-4175"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifan Chen","raw_affiliation_strings":["DCST, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-5364-4175","affiliations":[{"raw_affiliation_string":"DCST, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5118969370","display_name":"Hu YiRan","orcid":null},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hu YiRan","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-7986-3692","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089655391","display_name":"Qingyao Ai","orcid":"https://orcid.org/0000-0002-5030-709X"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingyao Ai","raw_affiliation_strings":["Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5030-709X","affiliations":[{"raw_affiliation_string":"Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Junjie Chen","orcid":"https://orcid.org/0009-0001-8883-9673"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junjie Chen","raw_affiliation_strings":["Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0001-8883-9673","affiliations":[{"raw_affiliation_string":"Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xiaoyu Yang","orcid":"https://orcid.org/0009-0007-1521-1360"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyu Yang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-1521-1360","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jianhui Yang","orcid":"https://orcid.org/0009-0004-3547-0472"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jianhui Yang","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0004-3547-0472","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041525078","display_name":"Yueyue Wu","orcid":"https://orcid.org/0000-0002-2539-8954"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yueyue Wu","raw_affiliation_strings":["Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2539-8954","affiliations":[{"raw_affiliation_string":"Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115596031","display_name":"Zeyang Liu","orcid":"https://orcid.org/0000-0001-6849-8675"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyang Liu","raw_affiliation_strings":["Quan Cheng Laboratory, Shandong, China and Shandong University, Shandong, China"],"raw_orcid":"https://orcid.org/0000-0001-6849-8675","affiliations":[{"raw_affiliation_string":"Quan Cheng Laboratory, Shandong, China and Shandong University, Shandong, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100668121","display_name":"Yiqun Liu","orcid":"https://orcid.org/0000-0002-0140-4512"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiqun Liu","raw_affiliation_strings":["Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0140-4512","affiliations":[{"raw_affiliation_string":"Quan Cheng Laboratory, Shandong, China and DCST, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3606","last_page":"3615"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9914000034332275,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.98580002784729,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8306783437728882},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.7979633212089539},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6268101930618286},{"id":"https://openalex.org/keywords/turn-taking","display_name":"Turn-taking","score":0.554395318031311},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3807220757007599},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.37418466806411743},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.36274322867393494},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.13867521286010742},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.1138589084148407},{"id":"https://openalex.org/keywords/communication","display_name":"Communication","score":0.0645608901977539}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8306783437728882},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.7979633212089539},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6268101930618286},{"id":"https://openalex.org/C2776352735","wikidata":"https://www.wikidata.org/wiki/Q2313343","display_name":"Turn-taking","level":3,"score":0.554395318031311},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3807220757007599},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.37418466806411743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.36274322867393494},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.13867521286010742},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.1138589084148407},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.0645608901977539},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3726302.3730340","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730340","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3726302.3730340","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3726302.3730340","pdf_url":null,"source":null,"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":16,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W2600463316","https://openalex.org/W2998702515","https://openalex.org/W3015322406","https://openalex.org/W3021016503","https://openalex.org/W4233192870","https://openalex.org/W4252076394","https://openalex.org/W4287795696","https://openalex.org/W4290742115","https://openalex.org/W4366999416","https://openalex.org/W4384641500","https://openalex.org/W4387321091","https://openalex.org/W4399058938","https://openalex.org/W4400528385","https://openalex.org/W4401857375","https://openalex.org/W4404783305"],"related_works":["https://openalex.org/W2228992124","https://openalex.org/W2613954729","https://openalex.org/W3130118443","https://openalex.org/W1955947659","https://openalex.org/W2759378137","https://openalex.org/W2376974064","https://openalex.org/W3188579947","https://openalex.org/W4289529495","https://openalex.org/W4386695291","https://openalex.org/W2355699586"],"abstract_inverted_index":{"Retrieval-augmented":[0],"generation":[1],"(RAG)":[2],"has":[3],"proven":[4],"highly":[5],"effective":[6,149],"in":[7,29,36,169,186],"improving":[8],"large":[9],"language":[10],"models":[11],"(LLMs)":[12],"across":[13],"various":[14,155],"domains.":[15],"However,":[16],"there":[17],"is":[18,71],"no":[19],"benchmark":[20,48,178],"specifically":[21],"designed":[22],"to":[23,49,145],"assess":[24],"the":[25,30,46,135,162,180,187],"effectiveness":[26],"of":[27,59,78,81,96,129,154,165,183],"RAG":[28,51,122,130,167,184],"legal":[31,55,67,74,98,121,136,171,188],"domain,":[32,189],"which":[33],"restricts":[34],"progress":[35],"this":[37,41],"area.":[38],"To":[39,113],"fill":[40],"gap,":[42],"we":[43,117,139,160],"propose":[44],"LexRAG,":[45],"first":[47],"evaluate":[50],"systems":[52,168,185],"for":[53,134,179],"multi-turn":[54,61,102],"consultations.":[56],"LexRAG":[57,84,174],"consists":[58,77],"1,013":[60],"dialogue":[62],"samples":[63],"and":[64,76,148,157,193],"17,228":[65],"candidate":[66],"articles.":[68],"Each":[69],"sample":[70],"annotated":[72],"by":[73],"experts":[75],"five":[79],"rounds":[80],"progressive":[82],"questioning.":[83],"includes":[85],"two":[86],"key":[87,163],"tasks:":[88],"(1)":[89],"Conversational":[90],"knowledge":[91],"retrieval,":[92],"requiring":[93],"accurate":[94],"retrieval":[95,158],"relevant":[97],"articles":[99],"based":[100],"on":[101,108],"context.":[103],"(2)":[104],"Response":[105],"generation,":[106],"focusing":[107],"producing":[109],"legally":[110],"sound":[111],"answers.":[112],"ensure":[114],"reliable":[115],"reproducibility,":[116],"develop":[118],"LexiT,":[119],"a":[120,126,176],"toolkit":[123],"that":[124],"provides":[125],"comprehensive":[127],"implementation":[128],"system":[131],"components":[132],"tailored":[133],"domain.":[137],"Additionally,":[138],"introduce":[140],"an":[141],"LLM-as-a-judge":[142],"evaluation":[143],"pipeline":[144],"enable":[146],"detailed":[147],"assessment.":[150],"Through":[151],"experimental":[152],"analysis":[153],"LLMs":[156],"methods,":[159],"reveal":[161],"limitations":[164],"existing":[166],"handling":[170],"consultation":[172],"conversations.":[173],"establishes":[175],"new":[177],"practical":[181],"application":[182],"with":[190],"its":[191],"code":[192],"data":[194],"available":[195],"at":[196],"https://github.com/CSHaitao/LexRAG.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":10},{"year":2025,"cited_by_count":3}],"updated_date":"2026-07-16T13:24:37.021932","created_date":"2025-10-10T00:00:00"}