{"id":"https://openalex.org/W7161836534","doi":"https://doi.org/10.48550/arxiv.2605.19815","title":"LP-Eval: Rubric and Dataset for Measuring the Quality of Legal Proposition Generation","display_name":"LP-Eval: Rubric and Dataset for Measuring the Quality of Legal Proposition Generation","publication_year":2026,"publication_date":"2026-05-19","ids":{"openalex":"https://openalex.org/W7161836534","doi":"https://doi.org/10.48550/arxiv.2605.19815"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.19815","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.19815","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101831874","display_name":"Shanshan Xu","orcid":"https://orcid.org/0009-0005-1203-1634"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Shanshan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088097035","display_name":"Johan Lindholm","orcid":"https://orcid.org/0000-0001-6009-7412"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lindholm, Johan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5131147383","display_name":"Amogh Raina","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Raina, Amogh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035763854","display_name":"Henrik Palmer Olsen","orcid":"https://orcid.org/0000-0001-8486-8752"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Olsen, Henrik Palmer","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5080674921","display_name":"Daniel Hershcovich","orcid":"https://orcid.org/0000-0002-3966-8708"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hershcovich, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.2370000034570694,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.2370000034570694,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.16449999809265137,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14013","display_name":"Legal Language and Interpretation","score":0.1573999971151352,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.9581999778747559},{"id":"https://openalex.org/keywords/proposition","display_name":"Proposition","score":0.8646000027656555},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.595300018787384},{"id":"https://openalex.org/keywords/economic-justice","display_name":"Economic Justice","score":0.44190001487731934},{"id":"https://openalex.org/keywords/european-union","display_name":"European union","score":0.43230000138282776},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.3750999867916107}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9581999778747559},{"id":"https://openalex.org/C2777152325","wikidata":"https://www.wikidata.org/wiki/Q108163","display_name":"Proposition","level":2,"score":0.8646000027656555},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.595300018787384},{"id":"https://openalex.org/C139621336","wikidata":"https://www.wikidata.org/wiki/Q3190382","display_name":"Economic Justice","level":2,"score":0.44190001487731934},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.4417000114917755},{"id":"https://openalex.org/C2910001868","wikidata":"https://www.wikidata.org/wiki/Q458","display_name":"European union","level":2,"score":0.43230000138282776},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.3750999867916107},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3598000109195709},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3521000146865845},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3260999917984009},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.29829999804496765},{"id":"https://openalex.org/C100701532","wikidata":"https://www.wikidata.org/wiki/Q5374244","display_name":"Empirical legal studies","level":3,"score":0.2863999903202057},{"id":"https://openalex.org/C55587333","wikidata":"https://www.wikidata.org/wiki/Q1133029","display_name":"Engineering ethics","level":1,"score":0.2784999907016754},{"id":"https://openalex.org/C143515210","wikidata":"https://www.wikidata.org/wiki/Q2479662","display_name":"Legal psychology","level":2,"score":0.27639999985694885},{"id":"https://openalex.org/C2993995455","wikidata":"https://www.wikidata.org/wiki/Q3150005","display_name":"Legal document","level":2,"score":0.2718999981880188},{"id":"https://openalex.org/C522695570","wikidata":"https://www.wikidata.org/wiki/Q6517578","display_name":"Legal research","level":2,"score":0.26989999413490295},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.2614000141620636},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.25600001215934753},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.25130000710487366}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.19815","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.19815","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.19815","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.7703648209571838,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Legal":[0,15],"proposition":[1,58],"generation":[2,22],"is":[3],"central":[4],"to":[5,136],"legal":[6,26,53,57,80],"reasoning":[7],"and":[8,23,63,91,117],"doctrinal":[9],"scholarship,":[10],"yet":[11],"remain":[12,134],"under-examined":[13],"in":[14],"NLP.":[16],"This":[17],"paper":[18],"investigates":[19],"the":[20,31,36],"automatic":[21],"evaluation":[24,49],"of":[25,30,33,35,73],"propositions":[27,101],"from":[28,103,108],"decisions":[29],"Court":[32],"Justice":[34],"European":[37],"Union":[38],"using":[39],"large":[40],"language":[41],"models":[42],"(LLMs).":[43],"We":[44,111],"introduce":[45],"LP-Eval,":[46],"a":[47,71],"three-step":[48],"rubric":[50],"co-designed":[51],"with":[52,126],"experts":[54],"that":[55,85,119],"decomposes":[56],"quality":[59,99],"into":[60],"formal":[61],"validity":[62],"substantive":[64],"dimensions.":[65],"Using":[66],"this":[67],"rubric,":[68],"we":[69],"release":[70],"dataset":[72],"two":[74],"experts'":[75],"annotations":[76],"for":[77,100],"100":[78],"LLM-generated":[79],"propositions.":[81],"Our":[82],"results":[83],"show":[84],"LLMs":[86,114],"can":[87],"generate":[88],"predominantly":[89],"well-formed":[90],"high-quality":[92],"propositions,":[93],"while":[94],"expert":[95,127],"evaluations":[96],"reveal":[97],"higher":[98],"derived":[102],"well":[104],"established":[105],"cases":[106],"than":[107,129],"recent":[109],"ones.":[110],"further":[112],"examine":[113],"as":[115],"evaluators":[116],"find":[118],"rubric-guided":[120],"LLM":[121],"judgments":[122],"align":[123],"more":[124],"closely":[125],"assessments":[128],"direct":[130],"overall":[131],"scoring,":[132],"but":[133],"insensitive":[135],"finer-grained":[137],"distinctions":[138],"captured":[139],"by":[140],"human":[141],"experts.":[142]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-21T00:00:00"}