{"id":"https://openalex.org/W7166663127","doi":"https://doi.org/10.48550/arxiv.2606.29920","title":"Can LLM-as-a-Judge Reliably Verify Rubrics in Agentic Scenarios?","display_name":"Can LLM-as-a-Judge Reliably Verify Rubrics in Agentic Scenarios?","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166663127","doi":"https://doi.org/10.48550/arxiv.2606.29920"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.29920","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29920","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.29920","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000864895","display_name":"\u5f6d\u626c\u8fbe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Yangda","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113281911","display_name":"Yunjia Qi","orcid":"https://orcid.org/0009-0009-1836-8413"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Yunjia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139676938","display_name":"Hao Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139648344","display_name":"Haotian Xia","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xia, Haotian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006809470","display_name":"Guanzhong He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Guanzhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106943140","display_name":"Xintong Shi","orcid":"https://orcid.org/0009-0008-8913-3905"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Xintong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126083297","display_name":"Richeng Xuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuan, Richeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108096359","display_name":"S Lu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Songyuanyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139672288","display_name":"Yixian Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yixian","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139634533","display_name":"Zhichao Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Zhichao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139690952","display_name":"Yuhong Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yuhong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139702642","display_name":"Lei Hou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Lei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139650224","display_name":"Bin Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Bin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139678310","display_name":"Juanzi Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Juanzi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.4205000102519989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.4205000102519989,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.2531999945640564,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.08829999715089798,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/rubric","display_name":"Rubric","score":0.9855999946594238},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6187000274658203},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5885000228881836},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5059999823570251},{"id":"https://openalex.org/keywords/voting","display_name":"Voting","score":0.40070000290870667},{"id":"https://openalex.org/keywords/component","display_name":"Component (thermodynamics)","score":0.3792000114917755}],"concepts":[{"id":"https://openalex.org/C111640148","wikidata":"https://www.wikidata.org/wiki/Q847349","display_name":"Rubric","level":2,"score":0.9855999946594238},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6466000080108643},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6187000274658203},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5885000228881836},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5059999823570251},{"id":"https://openalex.org/C520049643","wikidata":"https://www.wikidata.org/wiki/Q189760","display_name":"Voting","level":3,"score":0.40070000290870667},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39419999718666077},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38679999113082886},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.385699987411499},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.3792000114917755},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3619999885559082},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3052999973297119},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.273499995470047},{"id":"https://openalex.org/C153668964","wikidata":"https://www.wikidata.org/wiki/Q27636","display_name":"Majority rule","level":2,"score":0.26409998536109924},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2508000135421753}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.29920","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29920","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.29920","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.29920","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Rubric-based":[0],"scoring":[1,25],"has":[2],"become":[3],"a":[4,49,90,93,96,162],"widely":[5],"used":[6],"paradigm":[7],"in":[8,33,68],"model":[9],"evaluation,":[10],"typically":[11],"with":[12,85],"LLM-as-a-Judge":[13],"(LaaJ)":[14],"for":[15,23,55,64,71],"rubric":[16,24,56,69,146],"scoring.":[17,43],"However,":[18],"the":[19,61,101,104,117,132],"reliability":[20,54,67],"of":[21,52,134],"LaaJ":[22,53,66,136],"remains":[26],"underexplored.":[27],"This":[28],"concern":[29],"is":[30],"especially":[31],"pronounced":[32],"agentic":[34,72,78,83],"scenarios,":[35],"where":[36],"long,":[37],"complex":[38],"outputs":[39],"further":[40,130],"challenge":[41],"reliable":[42],"To":[44],"address":[45],"this,":[46],"we":[47,108],"conduct":[48],"systematic":[50],"meta-evaluation":[51],"verification.":[57,147],"We":[58,129,148,176],"introduce":[59],"RuVerBench,":[60,107],"first":[62],"benchmark":[63],"assessing":[65],"verification":[70,160],"scenarios.":[73],"RuVerBench":[74],"covers":[75],"two":[76],"prevalent":[77],"domains,":[79],"deep":[80],"research":[81],"and":[82,95,113,142,166,168,181],"coding,":[84],"2,458":[86],"instances,":[87],"each":[88],"containing":[89],"model-generated":[91],"output,":[92],"rubric,":[94],"human-annotated":[97],"label":[98],"indicating":[99],"whether":[100],"output":[102],"satisfies":[103],"rubric.":[105],"Using":[106],"evaluate":[109],"numerous":[110],"frontier":[111],"LLMs":[112],"find":[114,149],"that":[115,150],"even":[116],"most":[118],"advanced":[119],"models":[120,152],"achieve":[121],"strong":[122],"performance":[123],"but":[124,173],"still":[125],"exhibit":[126],"substantial":[127],"noise.":[128],"analyze":[131],"impact":[133],"key":[135],"strategies,":[137],"including":[138],"prompt":[139,157],"design,":[140],"batching,":[141],"majority":[143,169],"voting,":[144],"on":[145],"weaker":[151],"are":[153],"more":[154],"sensitive":[155],"to":[156,183],"variations,":[158],"batched":[159],"presents":[161],"trade-off":[163],"between":[164],"accuracy":[165],"efficiency,":[167],"voting":[170],"yields":[171],"effective":[172],"diminishing":[174],"returns.":[175],"have":[177],"released":[178],"our":[179],"dataset":[180],"code":[182],"facilitate":[184],"future":[185],"research:":[186],"https://github.com/THU-KEG/RuVerBench.":[187]},"counts_by_year":[],"updated_date":"2026-07-01T06:29:00.853634","created_date":"2026-07-01T00:00:00"}