{"id":"https://openalex.org/W7166852417","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1269","title":"AJ-Bench: Benchmarking Agent-as-a-Judge for Environment-Aware Evaluation","display_name":"AJ-Bench: Benchmarking Agent-as-a-Judge for Environment-Aware Evaluation","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166852417","doi":"https://doi.org/10.18653/v1/2026.findings-acl.1269"},"language":null,"primary_location":{"id":"doi:10.18653/v1/2026.findings-acl.1269","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1269","pdf_url":"https://aclanthology.org/2026.findings-acl.1269.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2026.findings-acl.1269.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139760540","display_name":"Wentao Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wentao Shi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139793138","display_name":"Yu Guang Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu Wang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139823129","display_name":"Yuyang Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuyang Zhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139849244","display_name":"Yuxin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuxin Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051925942","display_name":"Fuli Feng","orcid":"https://orcid.org/0000-0002-5828-9842"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fuli Feng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139846753","display_name":"Xueyuan Hao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xueyuan Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139749063","display_name":"Xi Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xi Su","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139791961","display_name":"Qi GU","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi GU","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139784476","display_name":"Hui Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hui Su","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139817245","display_name":"Xunliang Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xunliang Cai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139804676","display_name":"Xiangnan He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiangnan He","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.84176508,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"25371","last_page":"25413"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.22519999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.22519999742507935,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.05609999969601631,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12203","display_name":"Mobile Agent-Based Network Management","score":0.05420000106096268,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5105999708175659},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.4000000059604645},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2540999948978424},{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.24959999322891235},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.24480000138282776}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5472000241279602},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5105999708175659},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4000000059604645},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2540999948978424},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.24959999322891235},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.24940000474452972},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.24480000138282776},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.24449999630451202},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.24410000443458557},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2409999966621399}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.findings-acl.1269","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1269","pdf_url":"https://aclanthology.org/2026.findings-acl.1269.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.findings-acl.1269","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.findings-acl.1269","pdf_url":"https://aclanthology.org/2026.findings-acl.1269.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: ACL 2026","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.4391208291053772,"display_name":"Life in Land","id":"https://metadata.un.org/sdg/15"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166852417.pdf","grobid_xml":"https://content.openalex.org/works/W7166852417.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"reinforcement":[1],"learning":[2],"continues":[3],"to":[4,34,49,62],"scale":[5],"the":[6],"training":[7],"of":[8],"large":[9],"language":[10],"model-based":[11],"agents,":[12],"reliably":[13],"verifying":[14],"agent":[15],"behaviors":[16],"in":[17,87,108],"complex":[18],"environments":[19,46],"has":[20],"become":[21],"increasingly":[22],"challenging.Existing":[23],"approaches":[24],"rely":[25],"on":[26],"rule-based":[27],"verifiers":[28],"or":[29],"LLM-as-a-Judge":[30,100],"models,":[31],"which":[32],"struggle":[33],"generalize":[35],"beyond":[36],"narrow":[37],"domains.Agent-as-a-Judge":[38],"addresses":[39],"this":[40],"limitation":[41],"by":[42],"actively":[43],"interacting":[44],"with":[45],"and":[47,71,77,92,112],"tools":[48],"acquire":[50],"verifiable":[51],"evidence,":[52],"yet":[53],"its":[54],"capabilities":[55],"remain":[56],"underexplored.We":[57],"introduce":[58],"a":[59],"benchmark":[60,81],"AJ-Bench":[61],"systematically":[63],"evaluate":[64],"Agent-as-a-Judge":[65],"across":[66],"three":[67],"domains-search,":[68],"data":[69,111],"systems,":[70],"graphical":[72],"user":[73],"interfaces-comprising":[74],"155":[75],"tasks":[76],"516":[78],"annotated":[79],"trajectories.The":[80],"comprehensively":[82],"assesses":[83],"judge":[84],"agents'":[85],"abilities":[86],"information":[88],"acquisition,":[89],"state":[90],"verification,":[91],"process":[93],"verification.Experiments":[94],"demonstrate":[95],"consistent":[96],"performance":[97],"gains":[98],"over":[99],"baselines,":[101],"while":[102],"also":[103],"revealing":[104],"substantial":[105],"open":[106],"challenges":[107],"agent-based":[109],"verification.Our":[110],"code":[113],"are":[114],"available":[115],"at":[116],"https://aj-bench.github.io/.":[117]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-02T00:00:00"}