{"id":"https://openalex.org/W7143867236","doi":"https://doi.org/10.20736/0002002106","title":"Structured Evaluation of Legal Reasoning in LLMs: Chain-of-Thought Prompting and Human Scoring for Retrieval Robustness","display_name":"Structured Evaluation of Legal Reasoning in LLMs: Chain-of-Thought Prompting and Human Scoring for Retrieval Robustness","publication_year":2025,"publication_date":"2025-06-06","ids":{"openalex":"https://openalex.org/W7143867236","doi":"https://doi.org/10.20736/0002002106"},"language":"en","primary_location":{"id":"pmh:oai:irdb.nii.ac.jp:03100:0006839269","is_oa":true,"landing_page_url":"https://repository.nii.ac.jp/records/2002106","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},"type":"article","indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://repository.nii.ac.jp/records/2002106","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5131117326","display_name":"Ying-Chu Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Ying-Chu Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062379294","display_name":"Sieh-Chuen Huang","orcid":"https://orcid.org/0000-0003-3571-5236"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sieh-Chuen Huang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5071499141","display_name":"Hsuan-Lei Shao","orcid":"https://orcid.org/0000-0002-7101-5272"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hsuan-Lei Shao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5131117326"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.91771182,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"none","last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.24220000207424164,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.24220000207424164,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08940000087022781,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13910","display_name":"Computational and Text Analysis Methods","score":0.08529999852180481,"subfield":{"id":"https://openalex.org/subfields/3300","display_name":"General Social Sciences"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/statutory-law","display_name":"Statutory law","score":0.5922999978065491},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.527999997138977},{"id":"https://openalex.org/keywords/legal-psychology","display_name":"Legal psychology","score":0.49160000681877136},{"id":"https://openalex.org/keywords/legal-research","display_name":"Legal research","score":0.46380001306533813},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.4059999883174896},{"id":"https://openalex.org/keywords/legal-writing","display_name":"Legal writing","score":0.39500001072883606},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3946000039577484},{"id":"https://openalex.org/keywords/citation","display_name":"Citation","score":0.38429999351501465}],"concepts":[{"id":"https://openalex.org/C158129432","wikidata":"https://www.wikidata.org/wiki/Q7766927","display_name":"Statutory law","level":2,"score":0.5922999978065491},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5806999802589417},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.527999997138977},{"id":"https://openalex.org/C143515210","wikidata":"https://www.wikidata.org/wiki/Q2479662","display_name":"Legal psychology","level":2,"score":0.49160000681877136},{"id":"https://openalex.org/C522695570","wikidata":"https://www.wikidata.org/wiki/Q6517578","display_name":"Legal research","level":2,"score":0.46380001306533813},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.4059999883174896},{"id":"https://openalex.org/C2776502561","wikidata":"https://www.wikidata.org/wiki/Q1713997","display_name":"Legal writing","level":3,"score":0.39500001072883606},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39480000734329224},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3946000039577484},{"id":"https://openalex.org/C2778805511","wikidata":"https://www.wikidata.org/wiki/Q1713","display_name":"Citation","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C170692843","wikidata":"https://www.wikidata.org/wiki/Q15987302","display_name":"Legal profession","level":2,"score":0.3433000147342682},{"id":"https://openalex.org/C162040827","wikidata":"https://www.wikidata.org/wiki/Q126842","display_name":"Philosophy of law","level":3,"score":0.3301999866962433},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.326200008392334},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3172999918460846},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.3066999912261963},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3025999963283539},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2809999883174896},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.2784000039100647},{"id":"https://openalex.org/C2781310500","wikidata":"https://www.wikidata.org/wiki/Q1231428","display_name":"Persuasion","level":2,"score":0.26030001044273376},{"id":"https://openalex.org/C100701532","wikidata":"https://www.wikidata.org/wiki/Q5374244","display_name":"Empirical legal studies","level":3,"score":0.25999999046325684},{"id":"https://openalex.org/C20162079","wikidata":"https://www.wikidata.org/wiki/Q1151406","display_name":"Case-based reasoning","level":2,"score":0.25949999690055847}],"mesh":[],"locations_count":1,"locations":[{"id":"pmh:oai:irdb.nii.ac.jp:03100:0006839269","is_oa":true,"landing_page_url":"https://repository.nii.ac.jp/records/2002106","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"}],"best_oa_location":{"id":"pmh:oai:irdb.nii.ac.jp:03100:0006839269","is_oa":true,"landing_page_url":"https://repository.nii.ac.jp/records/2002106","pdf_url":null,"source":{"id":"https://openalex.org/S7407056385","display_name":"Institutional Repositories DataBase (IRDB)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I184597095","host_organization_name":"National Institute of Informatics","host_organization_lineage":["https://openalex.org/I184597095"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference paper"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.4432796537876129}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"This":[0],"study":[1,144],"investigates":[2],"the":[3,22,121,182],"legal":[4,51,104,168,176,195],"reasoning":[5,105,117],"abilities":[6],"of":[7,24,184],"Large":[8],"Language":[9],"Models":[10],"(LLMs)":[11],"in":[12,155,188],"Taiwan\u2019s":[13],"Status":[14],"Law":[15],"(family":[16],"and":[17,20,37,53,61,74,80,94,116,133,166,170,193],"inheritance":[18],"law)":[19],"evaluates":[21],"effects":[23],"Chain-of-Thought":[25],"(CoT)":[26],"prompting":[27,83,101,186],"on":[28],"answer":[29],"quality.":[30],"Six":[31],"essay":[32,76],"questions":[33],"from":[34],"past":[35],"judicial":[36],"graduate":[38],"law":[39,92],"exams":[40],"were":[41,63],"decomposed":[42,69],"into":[43],"68":[44],"sub-questions":[45],"targeting":[46],"issue":[47,111],"spotting,":[48],"statutory":[49,113],"application,":[50],"reasoning,":[52],"property":[54],"calculation.":[55],"Four":[56],"LLMs":[57],"(ChatGPT-4o,":[58],"Gemini,":[59],"Copilot,":[60],"Grok3)":[62],"evaluated":[64],"using":[65],"a":[66,91,95,159],"two-stage":[67],"framework:":[68],"sub-question":[70],"accuracy":[71,124],"(Stage":[72,84],"1)":[73],"full-length":[75],"response":[77],"performance":[78],"with":[79,86],"without":[81],"CoT":[82,100],"2),":[85],"human":[87,156],"scoring":[88],"conducted":[89],"by":[90,150],"professor":[93],"student.":[96],"Results":[97],"show":[98],"that":[99],"consistently":[102],"improves":[103],"quality":[106],"across":[107],"models,":[108],"notably":[109],"enhancing":[110],"coverage,":[112],"citation":[114],"accuracy,":[115],"structure.":[118],"Gemini":[119],"achieved":[120],"most":[122],"significant":[123],"gains":[125],"(from":[126],"83.2%":[127],"to":[128,146],"94.5%,":[129],"p":[130],"<":[131],"0.05)":[132],"was":[134],"selected":[135],"for":[136,164,173],"detailed":[137],"qualitative":[138],"analysis.":[139],"Beyond":[140],"model-specific":[141],"findings,":[142],"this":[143],"contributes":[145],"retrieval":[147],"evaluation":[148,161,197],"research":[149],"addressing":[151],"statistical":[152],"consistency":[153],"challenges":[154],"scoring,":[157],"proposing":[158],"diagnostic":[160],"method":[162],"adaptable":[163],"multilingual":[165],"multimedia":[167],"corpora,":[169],"suggesting":[171],"extensions":[172],"evaluating":[174],"enterprise-level":[175],"information":[177],"systems.":[178],"These":[179],"findings":[180],"underscore":[181],"value":[183],"structured":[185],"strategies":[187],"supporting":[189],"more":[190],"interpretable,":[191],"transferable,":[192],"scalable":[194],"AI":[196],"frameworks.":[198]},"counts_by_year":[],"updated_date":"2026-04-28T14:05:53.105641","created_date":"2026-04-01T00:00:00"}