{"id":"https://openalex.org/W7160958837","doi":"https://doi.org/10.48550/arxiv.2605.08437","title":"Magis-Bench: Evaluating LLMs on Magistrate-Level Legal Tasks","display_name":"Magis-Bench: Evaluating LLMs on Magistrate-Level Legal Tasks","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160958837","doi":"https://doi.org/10.48550/arxiv.2605.08437"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.08437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.08437","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5086873271","display_name":"Ramon Pires","orcid":"https://orcid.org/0000-0002-0023-1971"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pires, Ramon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135333581","display_name":"Thales Sales Almeida","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Almeida, Thales Sales","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135916764","display_name":"Celio Larcher Junior","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junior, Celio Larcher","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135917863","display_name":"Giovana Bon\u00e1s","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bon\u00e1s, Giovana","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135389386","display_name":"Hugo Abonizio","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abonizio, Hugo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045896508","display_name":"Marcos Piau","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Piau, Marcos","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135975811","display_name":"Roseval Malaquias Junior","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junior, Roseval Malaquias","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135362819","display_name":"Thiago Laitz","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laitz, Thiago","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135933499","display_name":"Rodrigo Nogueira","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nogueira, Rodrigo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.5748999714851379,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.5748999714851379,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.10220000147819519,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13904","display_name":"Artificial Intelligence Applications","score":0.033399999141693115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/legal-doctrine","display_name":"Legal doctrine","score":0.4661000072956085},{"id":"https://openalex.org/keywords/judicial-opinion","display_name":"Judicial opinion","score":0.42980000376701355},{"id":"https://openalex.org/keywords/legal-psychology","display_name":"Legal psychology","score":0.4108999967575073},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.3652999997138977},{"id":"https://openalex.org/keywords/doctrine","display_name":"Doctrine","score":0.3617999851703644},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.3517000079154968},{"id":"https://openalex.org/keywords/adjudication","display_name":"Adjudication","score":0.3481000065803528},{"id":"https://openalex.org/keywords/sophistication","display_name":"Sophistication","score":0.32839998602867126}],"concepts":[{"id":"https://openalex.org/C2778890055","wikidata":"https://www.wikidata.org/wiki/Q1192543","display_name":"Legal doctrine","level":2,"score":0.4661000072956085},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.4300000071525574},{"id":"https://openalex.org/C100505606","wikidata":"https://www.wikidata.org/wiki/Q6302997","display_name":"Judicial opinion","level":2,"score":0.42980000376701355},{"id":"https://openalex.org/C143515210","wikidata":"https://www.wikidata.org/wiki/Q2479662","display_name":"Legal psychology","level":2,"score":0.4108999967575073},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.40630000829696655},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3652999997138977},{"id":"https://openalex.org/C2776211767","wikidata":"https://www.wikidata.org/wiki/Q117850","display_name":"Doctrine","level":2,"score":0.3617999851703644},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.3517000079154968},{"id":"https://openalex.org/C204434341","wikidata":"https://www.wikidata.org/wiki/Q357789","display_name":"Adjudication","level":2,"score":0.3481000065803528},{"id":"https://openalex.org/C190253527","wikidata":"https://www.wikidata.org/wiki/Q295354","display_name":"Law and economics","level":1,"score":0.34299999475479126},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.33709999918937683},{"id":"https://openalex.org/C168725872","wikidata":"https://www.wikidata.org/wiki/Q991663","display_name":"Sophistication","level":2,"score":0.32839998602867126},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3212999999523163},{"id":"https://openalex.org/C2776035091","wikidata":"https://www.wikidata.org/wiki/Q7928819","display_name":"Viewpoints","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C100701532","wikidata":"https://www.wikidata.org/wiki/Q5374244","display_name":"Empirical legal studies","level":3,"score":0.3086000084877014},{"id":"https://openalex.org/C2778571376","wikidata":"https://www.wikidata.org/wiki/Q1355821","display_name":"Frontier","level":2,"score":0.30239999294281006},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.28859999775886536},{"id":"https://openalex.org/C538833194","wikidata":"https://www.wikidata.org/wiki/Q206937","display_name":"Civil procedure","level":2,"score":0.27889999747276306},{"id":"https://openalex.org/C73301696","wikidata":"https://www.wikidata.org/wiki/Q5469984","display_name":"Formalism (music)","level":3,"score":0.2775000035762787},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.27090001106262207},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.26980000734329224},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.26759999990463257},{"id":"https://openalex.org/C522695570","wikidata":"https://www.wikidata.org/wiki/Q6517578","display_name":"Legal research","level":2,"score":0.26460000872612},{"id":"https://openalex.org/C97364631","wikidata":"https://www.wikidata.org/wiki/Q484284","display_name":"Deductive reasoning","level":2,"score":0.26339998841285706},{"id":"https://openalex.org/C100521375","wikidata":"https://www.wikidata.org/wiki/Q2015382","display_name":"Competence (human resources)","level":2,"score":0.2563000023365021},{"id":"https://openalex.org/C205711294","wikidata":"https://www.wikidata.org/wiki/Q176953","display_name":"Rendering (computer graphics)","level":2,"score":0.25440001487731934},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.08437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.08437","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.08437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8100274801254272,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Existing":[0],"benchmarks":[1],"for":[2,54,67,168],"legal":[3,13,44,84,162,186],"AI":[4,187],"focus":[5],"primarily":[6],"on":[7,57,185],"tasks":[8,60],"where":[9],"LLMs":[10,56,107],"must":[11],"produce":[12],"arguments":[14,23],"or":[15],"documents,":[16],"yet":[17],"the":[18,94,137,150,157,173],"capacity":[19],"to":[20,30,41,181],"\\emph{judge}":[21],"such":[22],"--":[24,36],"weighing":[25],"competing":[26],"claims,":[27],"applying":[28],"doctrine":[29],"facts,":[31],"and":[32,80,90,99,146,164,178],"rendering":[33],"reasoned":[34],"decisions":[35],"is":[37],"arguably":[38],"as":[39,46,117],"fundamental":[40],"a":[42,52],"well-functioning":[43],"system":[45],"advocacy":[47],"itself.":[48],"We":[49,103,171],"introduce":[50],"Magis-Bench,":[51],"benchmark":[53],"evaluating":[55],"magistrate-level":[58],"writing":[59,165],"derived":[61],"from":[62,74],"recent":[63],"Brazilian":[64],"competitive":[65],"examinations":[66,76],"judicial":[68,101],"positions.":[69],"Magis-Bench":[70],"comprises":[71],"74":[72],"questions":[73,86],"eight":[75],"conducted":[77],"between":[78],"2023":[79],"2025,":[81],"including":[82],"discursive":[83],"analysis":[85],"with":[87,112,133],"multi-turn":[88],"structure":[89],"practical":[91],"exercises":[92],"requiring":[93],"composition":[95],"of":[96,156],"complete":[97,174],"civil":[98],"criminal":[100],"sentences.":[102],"evaluate":[104],"23":[105],"state-of-the-art":[106],"using":[108],"an":[109],"LLM-as-a-judge":[110],"methodology":[111],"four":[113],"independent":[114],"frontier":[115],"models":[116,152],"evaluators.":[118],"Our":[119],"results":[120],"show":[121],"strong":[122],"inter-judge":[123],"agreement":[124],"(Kendall's":[125],"$W":[126],"=":[127],"0.984$;":[128],"pairwise":[129],"Kendall's":[130],"$\u03c4\\ge":[131],"0.897$),":[132],"Google's":[134],"Gemini-3-Pro-Preview":[135],"achieving":[136],"highest":[138],"average":[139],"score":[140,153],"(6.97/10),":[141],"followed":[142],"by":[143],"Gemini-3-Flash-Preview":[144],"(6.67)":[145],"Claude-4.5-Opus":[147],"(6.46).":[148],"Even":[149],"best-performing":[151],"below":[154],"70\\%":[155],"maximum,":[158],"indicating":[159],"that":[160],"judicial-level":[161],"reasoning":[163],"remain":[166],"challenging":[167],"current":[169],"LLMs.":[170],"release":[172],"benchmark,":[175],"model":[176],"outputs,":[177],"evaluation":[179],"code":[180],"support":[182],"further":[183],"research":[184],"capabilities.":[188]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-13T00:00:00"}