{"id":"https://openalex.org/W7155091852","doi":"https://doi.org/10.48550/arxiv.2604.18240","title":"AJ-Bench: Benchmarking Agent-as-a-Judge for Environment-Aware Evaluation","display_name":"AJ-Bench: Benchmarking Agent-as-a-Judge for Environment-Aware Evaluation","publication_year":2026,"publication_date":"2026-04-20","ids":{"openalex":"https://openalex.org/W7155091852","doi":"https://doi.org/10.48550/arxiv.2604.18240"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.18240","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18240","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.18240","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5134122018","display_name":"Wentao Shi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Wentao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100445217","display_name":"Yu Wang","orcid":"https://orcid.org/0000-0002-4788-8655"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134189012","display_name":"Yuyang Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yuyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134191124","display_name":"Yuxin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134195829","display_name":"Fuli Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Fuli","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048834949","display_name":"Xueyuan Hao","orcid":"https://orcid.org/0000-0001-6098-5794"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hao, Xueyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5123872414","display_name":"Xi Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Xi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134192018","display_name":"Qi Gu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gu, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134136301","display_name":"Hui Su","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Su, Hui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134175308","display_name":"Xunliang Cai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cai, Xunliang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5134171655","display_name":"Xiangnan He","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"He, Xiangnan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.25600001215934753,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.25600001215934753,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.13050000369548798,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.12210000306367874,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8414000272750854},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7391999959945679},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5831000208854675},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5741999745368958},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5066999793052673},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4431999921798706},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.4239000082015991}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8414000272750854},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7488999962806702},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7391999959945679},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5831000208854675},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5741999745368958},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5066999793052673},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.489300012588501},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4431999921798706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4375},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.4239000082015991},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.41339999437332153},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.38769999146461487},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3824999928474426},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.3188000023365021},{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.30979999899864197},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.27810001373291016},{"id":"https://openalex.org/C2781020372","wikidata":"https://www.wikidata.org/wiki/Q533093","display_name":"On the fly","level":2,"score":0.2734000086784363},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2628999948501587},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.25870001316070557},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.25049999356269836}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.18240","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18240","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.18240","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.18240","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"reinforcement":[1],"learning":[2],"continues":[3],"to":[4,35,51,65],"scale":[5],"the":[6],"training":[7],"of":[8],"large":[9],"language":[10],"model-based":[11],"agents,":[12],"reliably":[13],"verifying":[14],"agent":[15],"behaviors":[16],"in":[17,91,113],"complex":[18],"environments":[19,48],"has":[20],"become":[21],"increasingly":[22],"challenging.":[23],"Existing":[24],"approaches":[25],"rely":[26],"on":[27],"rule-based":[28],"verifiers":[29],"or":[30],"LLM-as-a-Judge":[31,105],"models,":[32],"which":[33],"struggle":[34],"generalize":[36],"beyond":[37],"narrow":[38],"domains.":[39],"Agent-as-a-Judge":[40,68],"addresses":[41],"this":[42],"limitation":[43],"by":[44],"actively":[45],"interacting":[46],"with":[47],"and":[49,74,80,96,118],"tools":[50],"acquire":[52],"verifiable":[53],"evidence,":[54],"yet":[55],"its":[56],"capabilities":[57],"remain":[58],"underexplored.":[59],"We":[60],"introduce":[61],"a":[62],"benchmark":[63,85],"AJ-Bench":[64],"systematically":[66],"evaluate":[67],"across":[69],"three":[70],"domains-search,":[71],"data":[72,117],"systems,":[73],"graphical":[75],"user":[76],"interfaces-comprising":[77],"155":[78],"tasks":[79],"516":[81],"annotated":[82],"trajectories.":[83],"The":[84],"comprehensively":[86],"assesses":[87],"judge":[88],"agents'":[89],"abilities":[90],"information":[92],"acquisition,":[93],"state":[94],"verification,":[95],"process":[97],"verification.":[98,115],"Experiments":[99],"demonstrate":[100],"consistent":[101],"performance":[102],"gains":[103],"over":[104],"baselines,":[106],"while":[107],"also":[108],"revealing":[109],"substantial":[110],"open":[111],"challenges":[112],"agent-based":[114],"Our":[116],"code":[119],"are":[120],"available":[121],"at":[122],"https://aj-bench.github.io/.":[123]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-22T00:00:00"}