{"id":"https://openalex.org/W7138123451","doi":"https://doi.org/10.48550/arxiv.2603.15483","title":"Talk, Evaluate, Diagnose: User-aware Agent Evaluation with Automated Error Analysis","display_name":"Talk, Evaluate, Diagnose: User-aware Agent Evaluation with Automated Error Analysis","publication_year":2026,"publication_date":"2026-03-16","ids":{"openalex":"https://openalex.org/W7138123451","doi":"https://doi.org/10.48550/arxiv.2603.15483"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.15483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.15483","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5129670599","display_name":"Penny Chong","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chong, Penny","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029422042","display_name":"Harshavardhan Sunil Abichandani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Abichandani, Harshavardhan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5122228782","display_name":"Jiyuan Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Jiyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129678246","display_name":"Atin Ghosh","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ghosh, Atin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129650928","display_name":"Min Pyae Moe","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Moe, Min Pyae","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5129658487","display_name":"Yifan Mai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mai, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5002373633","display_name":"Daniel Dahlmeier","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dahlmeier, Daniel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18389999866485596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.18389999866485596,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.09019999951124191,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.07999999821186066,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.710099995136261},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6011000275611877},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.5274999737739563},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.451200008392334},{"id":"https://openalex.org/keywords/scripting-language","display_name":"Scripting language","score":0.4397999942302704},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.40939998626708984},{"id":"https://openalex.org/keywords/conversation","display_name":"Conversation","score":0.38659998774528503},{"id":"https://openalex.org/keywords/multi-agent-system","display_name":"Multi-agent system","score":0.3865000009536743},{"id":"https://openalex.org/keywords/dialog-system","display_name":"Dialog system","score":0.32409998774528503}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8489999771118164},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.710099995136261},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6011000275611877},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5274999737739563},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.451200008392334},{"id":"https://openalex.org/C61423126","wikidata":"https://www.wikidata.org/wiki/Q187432","display_name":"Scripting language","level":2,"score":0.4397999942302704},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41499999165534973},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4099999964237213},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.40939998626708984},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40610000491142273},{"id":"https://openalex.org/C2777200299","wikidata":"https://www.wikidata.org/wiki/Q52943","display_name":"Conversation","level":2,"score":0.38659998774528503},{"id":"https://openalex.org/C41550386","wikidata":"https://www.wikidata.org/wiki/Q529909","display_name":"Multi-agent system","level":2,"score":0.3865000009536743},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.35339999198913574},{"id":"https://openalex.org/C190954187","wikidata":"https://www.wikidata.org/wiki/Q5270587","display_name":"Dialog system","level":3,"score":0.32409998774528503},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3160000145435333},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.311599999666214},{"id":"https://openalex.org/C5894958","wikidata":"https://www.wikidata.org/wiki/Q2297769","display_name":"Software agent","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.2987000048160553},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.289900004863739},{"id":"https://openalex.org/C95623464","wikidata":"https://www.wikidata.org/wiki/Q1096149","display_name":"Classifier (UML)","level":2,"score":0.2854999899864197},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.27959999442100525},{"id":"https://openalex.org/C2777286243","wikidata":"https://www.wikidata.org/wiki/Q5591926","display_name":"Grading (engineering)","level":2,"score":0.27480000257492065},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C74072328","wikidata":"https://www.wikidata.org/wiki/Q1142726","display_name":"Intelligent agent","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2542000114917755},{"id":"https://openalex.org/C169258074","wikidata":"https://www.wikidata.org/wiki/Q245748","display_name":"Random forest","level":2,"score":0.2533999979496002},{"id":"https://openalex.org/C4554734","wikidata":"https://www.wikidata.org/wiki/Q593744","display_name":"Knowledge base","level":2,"score":0.25270000100135803}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.15483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.15483","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.15483","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.4217291474342346,"display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Agent":[0],"applications":[1],"are":[2],"increasingly":[3],"adopted":[4],"to":[5,13,23,36,49],"automate":[6],"workflows":[7],"across":[8,207],"diverse":[9],"tasks.":[10],"However,":[11],"due":[12],"the":[14,50,65,71,77,105,161,164,178,181,231,236],"heterogeneous":[15],"domains":[16],"they":[17,59],"operate":[18],"in,":[19],"it":[20],"is":[21],"challenging":[22],"create":[24],"a":[25,53],"scalable":[26],"evaluation":[27,56,85],"framework.":[28],"Prior":[29],"works":[30],"each":[31],"employ":[32],"their":[33],"own":[34],"methods":[35],"determine":[37],"task":[38],"success,":[39],"such":[40],"as":[41,135],"database":[42],"lookups,":[43],"regex":[44],"match,":[45],"etc.,":[46],"adding":[47],"complexity":[48],"development":[51],"of":[52,97,160,180,223],"unified":[54],"agent":[55,84,98,162,193,205,219],"approach.":[57],"Moreover,":[58],"do":[60],"not":[61],"systematically":[62],"account":[63],"for":[64,123,192],"user's":[66],"role":[67],"nor":[68],"expertise":[69,211],"in":[70,218],"interaction,":[72],"providing":[73,189],"incomplete":[74],"insights":[75,203],"into":[76,235],"agent's":[78,237],"performance.":[79],"We":[80,113,128,148,169,195,213],"argue":[81],"that":[82,152,176,197],"effective":[83],"goes":[86],"beyond":[87],"correctness":[88],"alone,":[89],"incorporating":[90,230],"conversation":[91],"quality,":[92],"efficiency":[93,156],"and":[94,118,138,157,183,188,209],"systematic":[95],"diagnosis":[96],"errors.":[99],"To":[100],"address":[101],"this,":[102],"we":[103],"introduce":[104,170],"TED":[106,199],"framework":[107,200],"(Talk,":[108],"Evaluate,":[109],"Diagnose).":[110],"(1)":[111],"Talk:":[112],"leverage":[114],"reusable,":[115],"generic":[116],"expert":[117],"non-expert":[119],"user":[120,210],"persona":[121],"templates":[122],"user-agent":[124],"interaction.":[125],"(2)":[126],"Evaluate:":[127],"adapt":[129],"existing":[130],"datasets":[131],"by":[132],"representing":[133],"subgoals-such":[134],"tool":[136,175],"signatures,":[137],"responses-as":[139],"natural":[140],"language":[141],"grading":[142],"notes,":[143],"evaluated":[144],"automatically":[145],"with":[146,221],"LLM-as-a-judge.":[147],"propose":[149],"new":[150,202],"metrics":[151,228],"capture":[153],"both":[154],"turn":[155],"intermediate":[158],"progress":[159],"complementing":[163],"user-aware":[165],"setup.":[166],"(3)":[167],"Diagnose:":[168],"an":[171],"automated":[172],"error":[173,233],"analysis":[174],"analyzes":[177],"inconsistencies":[179],"judge":[182],"agents,":[184],"uncovering":[185],"common":[186],"errors,":[187],"actionable":[190],"feedback":[191],"improvement.":[194],"show":[196],"our":[198,226],"reveals":[201],"regarding":[204],"performance":[206,220],"models":[208],"levels.":[212],"also":[214],"demonstrate":[215],"potential":[216],"gains":[217],"peaks":[222],"8-10%":[224],"on":[225],"proposed":[227],"after":[229],"identified":[232],"remedies":[234],"design.":[238]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-18T00:00:00"}