{"id":"https://openalex.org/W7166841895","doi":"https://doi.org/10.18653/v1/2026.findings-acl.634","title":"JudgeAgent: Beyond Static Benchmarks for Knowledge-Driven and Dynamic LLM Evaluation","display_name":"JudgeAgent: Beyond Static Benchmarks for Knowledge-Driven and Dynamic LLM Evaluation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166841895","doi":"https://doi.org/10.18653/v1/2026.findings-acl.634"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.634","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.634","pdf_url":"https://aclanthology.org/2026.findings-acl.634.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.634.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139827301","display_name":"Zhichao Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhichao Shi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139770736","display_name":"Xuhui Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuhui Jiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139832605","display_name":"Chengjin Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chengjin Xu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139772082","display_name":"Cangli Yao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cangli Yao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139731242","display_name":"Shengjie Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shengjie Ma","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086972170","display_name":"Yinghan Shen","orcid":"https://orcid.org/0009-0004-1727-665X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yinghan Shen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139768938","display_name":"Zixuan Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zixuan Li","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139731084","display_name":"Jian Guo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian Guo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139821945","display_name":"Yuanzhuo Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuanzhuo Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.93451243,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"13004","last_page":"13030"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.522599995136261,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.522599995136261,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.029899999499320984,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.02979999966919422,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.2892000079154968},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2718000113964081},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.25929999351501465},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.2551000118255615},{"id":"https://openalex.org/keywords/troubleshooting","display_name":"Troubleshooting","score":0.2393999993801117}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5720999836921692},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.2892000079154968},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2718000113964081},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.267300009727478},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25929999351501465},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.25769999623298645},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2551000118255615},{"id":"https://openalex.org/C147494362","wikidata":"https://www.wikidata.org/wiki/Q2078905","display_name":"Troubleshooting","level":2,"score":0.2393999993801117},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.23880000412464142},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.23880000412464142}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.634","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.634","pdf_url":"https://aclanthology.org/2026.findings-acl.634.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.634","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.634","pdf_url":"https://aclanthology.org/2026.findings-acl.634.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166841895.pdf","grobid_xml":"https://content.openalex.org/works/W7166841895.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Current":[0],"evaluation":[1,53,128],"methods":[2],"for":[3,55,77],"large":[4],"language":[5],"models":[6],"(LLMs)":[7],"primarily":[8],"rely":[9],"on":[10,134],"static":[11],"benchmarks,":[12],"presenting":[13],"two":[14],"major":[15],"challenges:":[16],"limited":[17,61],"knowledge":[18,62,74],"coverage":[19],"and":[20,51,84,91,100,115,126],"fixed":[21],"difficulties":[22],"that":[23,109],"mismatch":[24],"with":[25,69],"the":[26,39,58,121],"evaluated":[27],"LLMs.These":[28],"limitations":[29],"lead":[30],"to":[31,72,80],"superficial":[32],"assessments":[33],"of":[34,60,105,123],"LLM":[35,66],"knowledge,":[36],"thereby":[37],"impeding":[38],"targeted":[40],"model":[41,118],"optimizations.To":[42],"bridge":[43],"this":[44,124],"gap,":[45],"we":[46],"propose":[47],"JudgeAgent,":[48],"a":[49,89],"knowledge-driven":[50,125],"dynamic":[52,127],"framework":[54],"LLMs.To":[56],"address":[57],"challenge":[59],"coverage,":[63],"JudgeAgent":[64,95,110],"leverages":[65],"agents":[67],"equipped":[68],"context":[70],"graphs":[71],"traverse":[73],"structures":[75],"systematically":[76],"question":[78],"generation.Furthermore,":[79],"mitigate":[81],"data":[82],"contamination":[83],"difficulty":[85],"mismatch,":[86],"it":[87],"adopts":[88],"difficulty-adaptive":[90],"multi-turn":[92],"interview":[93],"mechanism.Thereby,":[94],"can":[96],"achieve":[97],"comprehensive":[98,113],"evaluations":[99,114],"facilitate":[101],"more":[102,112],"effective":[103,117],"improvement":[104],"LLMs.Empirical":[106],"results":[107],"demonstrate":[108],"enables":[111],"facilitates":[116],"iterations,":[119],"highlighting":[120],"potential":[122],"paradigm.The":[129],"source":[130],"code":[131],"is":[132],"available":[133],"https://github.com/DataArcTech/":[135],"JudgeAgent.":[136]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}