{"id":"https://openalex.org/W7225253330","doi":"https://doi.org/10.5281/zenodo.23949613","title":"Exam questions are not consultations: on training and evaluating specialised legal language models with examination material, and an alternative evaluation design","display_name":"Exam questions are not consultations: on training and evaluating specialised legal language models with examination material, and an alternative evaluation design","publication_year":2026,"publication_date":"2026-11-04","ids":{"openalex":"https://openalex.org/W7225253330","doi":"https://doi.org/10.5281/zenodo.23949613"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.23949613","doi":"https://doi.org/10.5281/zenodo.23949613","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23949613","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.23949613","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5158244390","display_name":"Sergei Sevriukov","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Sevriukov, Sergei","raw_affiliation_strings":["LeckerMath"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LeckerMath","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":["https://openalex.org/A5158244390"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12755","display_name":"Legal Education and Practice Innovations","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/3308","display_name":"Law"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exam-questions","display_name":"exam questions","score":0.4650000035762787},{"id":"https://openalex.org/keywords/evaluation-design","display_name":"evaluation design","score":0.7630000114440918},{"id":"https://openalex.org/keywords/german-tax-law","display_name":"German tax law","score":0.7509999871253967},{"id":"https://openalex.org/keywords/legal-advice","display_name":"legal advice","score":0.5329999923706055},{"id":"https://openalex.org/keywords/legal-change","display_name":"legal change","score":0.3709999918937683}],"concepts":[{"id":"https://openalex.org/C154775046","wikidata":"https://www.wikidata.org/wiki/Q188","display_name":"German","level":2,"score":0.6007999777793884},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5181999802589417},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4853000044822693},{"id":"https://openalex.org/C98184364","wikidata":"https://www.wikidata.org/wiki/Q1780131","display_name":"Argument (complex analysis)","level":2,"score":0.47369998693466187},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.4690000116825104},{"id":"https://openalex.org/C191795146","wikidata":"https://www.wikidata.org/wiki/Q3878446","display_name":"Norm (philosophy)","level":2,"score":0.44690001010894775},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.3959999978542328},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.38940000534057617},{"id":"https://openalex.org/C522695570","wikidata":"https://www.wikidata.org/wiki/Q6517578","display_name":"Legal research","level":2,"score":0.37929999828338623},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.35899999737739563},{"id":"https://openalex.org/C2776502561","wikidata":"https://www.wikidata.org/wiki/Q1713997","display_name":"Legal writing","level":3,"score":0.34290000796318054},{"id":"https://openalex.org/C145420912","wikidata":"https://www.wikidata.org/wiki/Q853077","display_name":"Mathematics education","level":1,"score":0.3249000012874603},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.3116999864578247},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3068999946117401},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.2992999851703644},{"id":"https://openalex.org/C524211801","wikidata":"https://www.wikidata.org/wiki/Q865436","display_name":"Legal education","level":2,"score":0.2847000062465668},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.28049999475479126},{"id":"https://openalex.org/C170706310","wikidata":"https://www.wikidata.org/wiki/Q30216","display_name":"Common law","level":2,"score":0.2728999853134155},{"id":"https://openalex.org/C186528591","wikidata":"https://www.wikidata.org/wiki/Q619544","display_name":"Body of knowledge","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.26980000734329224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25920000672340393},{"id":"https://openalex.org/C2778890055","wikidata":"https://www.wikidata.org/wiki/Q1192543","display_name":"Legal doctrine","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.2581999897956848}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.5281/zenodo.23949613","doi":"https://doi.org/10.5281/zenodo.23949613","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23949613","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5281/zenodo.23949614","doi":"https://doi.org/10.5281/zenodo.23949614","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23949614","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.5281/zenodo.23949613","doi":"https://doi.org/10.5281/zenodo.23949613","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.23949613","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.5473999977111816,"display_name":"Peace, justice, and strong institutions","id":"https://metadata.un.org/sdg/16"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":1,"referenced_works":["https://openalex.org/W7225256559"],"related_works":["https://openalex.org/W7225231928","https://openalex.org/W7212047482","https://openalex.org/W7165735751","https://openalex.org/W6977525285","https://openalex.org/W7155696795","https://openalex.org/W7166848046","https://openalex.org/W7118227629","https://openalex.org/W7020540047","https://openalex.org/W7225098973","https://openalex.org/W7164048779"],"abstract_inverted_index":{"Examination":[0],"material":[1,33],"is":[2,125],"an":[3,46,122,190],"attractive":[4],"source":[5],"for":[6,38,127],"training":[7,142],"and":[8,21,35,131,143,146,182,208,246],"evaluating":[9],"specialised":[10],"legal":[11,129,252],"language":[12],"models:":[13],"it":[14],"comes":[15],"with":[16,65,185,197],"fixed":[17],"facts,":[18,53,67,199],"reference":[19],"solutions":[20],"human":[22],"score":[23],"distributions.":[24],"I":[25,188],"examine":[26],"what":[27],"a":[28,55,238],"design":[29,192,223],"built":[30],"on":[31,100,168,194],"such":[32,234],"can":[34],"cannot":[36],"measure":[37],"advisory":[39],"work":[40],"in":[41,61,89],"German":[42,247],"tax":[43,254],"law.":[44],"First,":[45],"exam":[47],"question":[48,214],"hands":[49],"over":[50,85],"all":[51],"relevant":[52],"whereas":[54],"consultation":[56],"starts":[57],"from":[58,180,205],"incomplete":[59],"facts;":[60],"my":[62,118],"own":[63,119],"games":[64,107],"hidden":[66,198],"68":[68],"%":[69,176,184],"of":[70,103,111,141,149,158,177,224,229,237],"lost":[71],"points":[72],"were":[73],"facts":[74,179],"the":[75,82,96,101,104,135,139,147,163,178,186,222,227,230],"adviser":[76],"did":[77],"not":[78,226],"ask":[79],"about.":[80],"Second,":[81],"law":[83,136],"changes":[84],"time;":[86],"knowledge":[87],"stored":[88],"model":[90,150,173],"weights":[91],"has":[92],"one":[93,115,128,172],"date,":[94],"while":[95],"correct":[97,216],"answer":[98,123],"depends":[99],"date":[102],"case.":[105],"My":[106],"contain":[108],"both":[109],"directions":[110],"this":[112],"error,":[113],"including":[114],"made":[116],"by":[117],"pipeline.":[120],"Third,":[121],"key":[124],"written":[126],"state":[130],"becomes":[132],"outdated":[133],"when":[134],"changes.":[137],"Fourth,":[138],"overlap":[140],"test":[144],"sources":[145],"use":[148],"judges":[151],"need":[152],"explicit":[153],"controls.":[154],"A":[155],"dated":[156],"anchor":[157],"official":[159],"norm":[160],"texts":[161],"addresses":[162],"second":[164],"point":[165],"without":[166],"retraining:":[167],"52":[169],"expert":[170],"questions,":[171],"found":[174],"35":[175],"memory":[181],"83":[183],"anchor.":[187],"propose":[189],"evaluation":[191],"based":[193],"fictitious":[195],"cases":[196],"blind":[200],"judges,":[201],"control":[202],"questions":[203],"drawn":[204],"random":[206],"norms":[207],"three":[209],"metrics:":[210],"decisive":[211],"fact":[212],"found,":[213],"asked,":[215],"version":[217,244,248],"cited.":[218],"The":[219],"argument":[220],"concerns":[221],"evaluation,":[225],"qualification":[228],"people":[231],"who":[232],"build":[233],"models.":[235],"Part":[236],"series;":[239],"overview:":[240],"https://doi.org/10.5281/zenodo.23949487.":[241],"Files:":[242],"English":[243],"(authoritative)":[245],"(Deutsche":[249],"Fassung).":[250],"Not":[251],"or":[253],"advice.":[255]},"counts_by_year":[],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2026-11-05T00:00:00"}