{"id":"https://openalex.org/W7165643980","doi":"https://doi.org/10.48550/arxiv.2606.21627","title":"Counsel: A Meta-Evaluation Dataset for Agentic Tasks","display_name":"Counsel: A Meta-Evaluation Dataset for Agentic Tasks","publication_year":2026,"publication_date":"2026-06-19","ids":{"openalex":"https://openalex.org/W7165643980","doi":"https://doi.org/10.48550/arxiv.2606.21627"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.21627","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21627","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.21627","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5116082777","display_name":"Sashank Pisupati","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Pisupati, Sashank","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116082780","display_name":"Henry Broomfield","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Broomfield, Henry","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139140314","display_name":"Eujeong Choi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Choi, Eujeong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139156765","display_name":"Antonia Calvi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Calvi, Antonia","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139198098","display_name":"Charlie Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Charlie","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5116082786","display_name":"Roman Engeler","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Engeler, Roman","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018864006","display_name":"Max Bartolo","orcid":"https://orcid.org/0009-0007-3301-7895"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bartolo, Max","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139151873","display_name":"Patrick Lewis","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lewis, Patrick","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.3172999918460846,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.3172999918460846,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.14429999887943268,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.11900000274181366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6201000213623047},{"id":"https://openalex.org/keywords/soundness","display_name":"Soundness","score":0.597100019454956},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5577999949455261},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.43650001287460327},{"id":"https://openalex.org/keywords/human-error","display_name":"Human error","score":0.4359000027179718},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.3465999960899353}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.671999990940094},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6201000213623047},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.597100019454956},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5577999949455261},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4717999994754791},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.43650001287460327},{"id":"https://openalex.org/C169806903","wikidata":"https://www.wikidata.org/wiki/Q5937752","display_name":"Human error","level":2,"score":0.4359000027179718},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35499998927116394},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3522000014781952},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.3465999960899353},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.28839999437332153},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.26910001039505005}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.21627","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21627","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.21627","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.21627","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/1","score":0.5143199563026428,"display_name":"No poverty"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"As":[0],"agentic":[1,24,84,232],"systems":[2],"tackle":[3],"increasingly":[4],"complex":[5],"multi-step":[6],"tasks,":[7],"evaluating":[8],"their":[9],"trajectories":[10],"presents":[11],"a":[12,19,155],"major":[13],"bottleneck":[14],"-":[15],"human":[16,107,148,189],"annotation":[17],"of":[18,67,81,88,109,139,228],"single":[20],"trajectory":[21,156],"on":[22,47,94,116,198,202],"popular":[23],"benchmarks":[25],"can":[26],"take":[27],"hours,":[28],"making":[29],"it":[30],"difficult":[31],"to":[32,54,164],"scale":[33],"evaluations":[34],"for":[35,83,170,214,231],"measuring":[36],"performance":[37],"or":[38,128,167],"curating":[39],"training":[40],"data.":[41],"This":[42],"has":[43],"driven":[44],"widespread":[45],"reliance":[46],"automated":[48],"approaches":[49],"such":[50],"as":[51,120,161],"LLM-as-a-judge":[52],"(LLMJ)":[53],"critique":[55],"agents":[56],"at":[57,62],"the":[58,65,77,192],"process":[59],"and":[60,102,106,157,182,200,210,225],"outcome-levels":[61],"scale,":[63],"however,":[64],"soundness":[66],"LLMJ":[68,145],"critiques":[69,90,115,146],"often":[70],"goes":[71],"unmeasured.":[72],"Here,":[73],"we":[74,175,219],"introduce":[75],"Counsel,":[76],"first":[78],"public":[79],"dataset":[80,143],"meta-evaluations":[82,108],"tasks.":[85],"Counsel":[86,204],"consists":[87],"process-level":[89],"from":[91],"open-weight":[92,173,208],"LLMJs":[93,169],"two":[95],"agent":[96],"benchmarks:":[97],"tau-bench":[98],"(customer":[99],"support":[100],"agents)":[101],"DA-Code":[103],"(coding":[104],"agents),":[105],"these":[110],"critiques.":[111],"Human":[112],"annotators":[113],"label":[114],"each":[117],"flagged":[118],"error":[119,152],"\"spot":[121],"on\",":[122],"\"correct":[123],"location":[124,153,199],"but":[125],"poor":[126],"reasoning\",":[127],"\"should":[129],"not":[130],"have":[131],"flagged\",":[132],"achieving":[133],"reliable":[134],"inter-annotator":[135],"agreement":[136,197],"(Krippendorff's":[137],"alpha":[138],"0.78).":[140],"The":[141],"resulting":[142],"stratifies":[144],"by":[147],"alignment":[149,227],"across":[150],"both":[151,186],"within":[154],"reasoning":[158,184],"quality,":[159],"serving":[160],"valuable":[162],"data":[163],"calibrate,":[165],"improve,":[166],"train":[168],"agents.":[171],"Comparing":[172],"judges,":[174],"find":[176],"that":[177],"more":[178,183],"capable":[179],"judge":[180,194],"models":[181,209],"effort":[185],"enabled":[187],"improved":[188,226],"agreement,":[190],"with":[191],"strongest":[193],"reaching":[195],"~88%":[196],"~65%":[201],"reasoning.":[203],"is":[205,211],"generated":[206],"using":[207],"permissively":[212],"licensed":[213],"broad":[215],"community":[216],"use,":[217],"which":[218],"hope":[220],"will":[221],"enable":[222],"rigorous":[223],"study":[224],"LLM-based":[229],"evaluators":[230],"systems.":[233]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-24T00:00:00"}