{"id":"https://openalex.org/W7128909499","doi":"https://doi.org/10.1145/3793671","title":"Evaluating LLM-based Agents for Multi-turn Conversations: A Survey","display_name":"Evaluating LLM-based Agents for Multi-turn Conversations: A Survey","publication_year":2026,"publication_date":"2026-02-14","ids":{"openalex":"https://openalex.org/W7128909499","doi":"https://doi.org/10.1145/3793671"},"language":"en","primary_location":{"id":"doi:10.1145/3793671","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3793671","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5125561829","display_name":"Shengyue Guan","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shengyue Guan","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0000-8321-6365","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Jindong Wang","orcid":"https://orcid.org/0000-0002-4833-0880"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jindong Wang","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-4833-0880","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055104982","display_name":"\u5c06\u5c1a \u6e21\u8fba","orcid":"https://orcid.org/0000-0002-6337-9375"},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiang Bian","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6337-9375","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126038066","display_name":"Bin Zhu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Zhu","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3571-7808","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126034557","display_name":"Jian-Guang Lou","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian-Guang Lou","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8496-033X","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5125616993","display_name":"Haoyi Xiong","orcid":null},"institutions":[{"id":"https://openalex.org/I4210113369","display_name":"Microsoft Research Asia (China)","ror":"https://ror.org/0300m5276","country_code":"CN","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210113369"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haoyi Xiong","raw_affiliation_strings":["Microsoft, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5451-3253","affiliations":[{"raw_affiliation_string":"Microsoft, Beijing, China","institution_ids":["https://openalex.org/I4210113369"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210113369"],"apc_list":null,"apc_paid":null,"fwci":30.3464,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.99460412,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"17","issue":"4","first_page":"1","last_page":"40"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.2766000032424927,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.2766000032424927,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.23270000517368317,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.15839999914169312,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/automatic-summarization","display_name":"Automatic summarization","score":0.6789000034332275},{"id":"https://openalex.org/keywords/taxonomy","display_name":"Taxonomy (biology)","score":0.6647999882698059},{"id":"https://openalex.org/keywords/blueprint","display_name":"Blueprint","score":0.6581000089645386},{"id":"https://openalex.org/keywords/bridge","display_name":"Bridge (graph theory)","score":0.5644000172615051},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4465999901294708},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4366999864578247},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.41909998655319214}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8896999955177307},{"id":"https://openalex.org/C170858558","wikidata":"https://www.wikidata.org/wiki/Q1394144","display_name":"Automatic summarization","level":2,"score":0.6789000034332275},{"id":"https://openalex.org/C58642233","wikidata":"https://www.wikidata.org/wiki/Q8269924","display_name":"Taxonomy (biology)","level":2,"score":0.6647999882698059},{"id":"https://openalex.org/C155911762","wikidata":"https://www.wikidata.org/wiki/Q422321","display_name":"Blueprint","level":2,"score":0.6581000089645386},{"id":"https://openalex.org/C100776233","wikidata":"https://www.wikidata.org/wiki/Q2532492","display_name":"Bridge (graph theory)","level":2,"score":0.5644000172615051},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4465999901294708},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4366999864578247},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.41909998655319214},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.41130000352859497},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.396699994802475},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.3407999873161316},{"id":"https://openalex.org/C174348530","wikidata":"https://www.wikidata.org/wiki/Q188635","display_name":"Bridging (networking)","level":2,"score":0.31610000133514404},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.30469998717308044},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.2678000032901764},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.26739999651908875},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.26030001044273376},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2515000104904175},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3793671","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3793671","pdf_url":null,"source":{"id":"https://openalex.org/S2492086750","display_name":"ACM Transactions on Intelligent Systems and Technology","issn_l":"2157-6904","issn":["2157-6904","2157-6912"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Intelligent Systems and Technology","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.736552894115448}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":137,"referenced_works":["https://openalex.org/W1518951372","https://openalex.org/W2024932032","https://openalex.org/W2058373514","https://openalex.org/W2101105183","https://openalex.org/W2105594594","https://openalex.org/W2133512280","https://openalex.org/W2160637580","https://openalex.org/W2740698270","https://openalex.org/W2903949349","https://openalex.org/W2944069152","https://openalex.org/W2952855649","https://openalex.org/W2960532269","https://openalex.org/W2963532001","https://openalex.org/W2963825865","https://openalex.org/W2964006684","https://openalex.org/W2972664115","https://openalex.org/W2972732298","https://openalex.org/W2985067290","https://openalex.org/W2995200518","https://openalex.org/W3002772880","https://openalex.org/W3010229169","https://openalex.org/W3011082615","https://openalex.org/W3035317050","https://openalex.org/W3098258760","https://openalex.org/W3110968911","https://openalex.org/W3114032074","https://openalex.org/W3114142812","https://openalex.org/W3127203257","https://openalex.org/W3156017614","https://openalex.org/W3186804217","https://openalex.org/W3207604732","https://openalex.org/W3211346883","https://openalex.org/W3213439133","https://openalex.org/W4205165244","https://openalex.org/W4220681721","https://openalex.org/W4236521339","https://openalex.org/W4283123655","https://openalex.org/W4283764330","https://openalex.org/W4285188834","https://openalex.org/W4285273040","https://openalex.org/W4287854583","https://openalex.org/W4287854961","https://openalex.org/W4287887196","https://openalex.org/W4290742115","https://openalex.org/W4293863350","https://openalex.org/W4306317408","https://openalex.org/W4310261394","https://openalex.org/W4319591929","https://openalex.org/W4323519309","https://openalex.org/W4383108457","https://openalex.org/W4385562575","https://openalex.org/W4385567026","https://openalex.org/W4385570019","https://openalex.org/W4385570326","https://openalex.org/W4385570827","https://openalex.org/W4385571232","https://openalex.org/W4385571408","https://openalex.org/W4385571842","https://openalex.org/W4385572023","https://openalex.org/W4385573049","https://openalex.org/W4385573628","https://openalex.org/W4385573801","https://openalex.org/W4385574004","https://openalex.org/W4385574184","https://openalex.org/W4385734161","https://openalex.org/W4387846321","https://openalex.org/W4388878454","https://openalex.org/W4389518608","https://openalex.org/W4389518874","https://openalex.org/W4389520103","https://openalex.org/W4389520177","https://openalex.org/W4389520226","https://openalex.org/W4389520732","https://openalex.org/W4389523655","https://openalex.org/W4389523711","https://openalex.org/W4389523826","https://openalex.org/W4389523925","https://openalex.org/W4389524315","https://openalex.org/W4390094929","https://openalex.org/W4390107999","https://openalex.org/W4391136507","https://openalex.org/W4393065402","https://openalex.org/W4393160302","https://openalex.org/W4396833218","https://openalex.org/W4401042177","https://openalex.org/W4401042350","https://openalex.org/W4401042703","https://openalex.org/W4401042793","https://openalex.org/W4401042826","https://openalex.org/W4401043154","https://openalex.org/W4401043434","https://openalex.org/W4401043616","https://openalex.org/W4401044184","https://openalex.org/W4401306603","https://openalex.org/W4402670877","https://openalex.org/W4402671320","https://openalex.org/W4402671584","https://openalex.org/W4402671805","https://openalex.org/W4402671949","https://openalex.org/W4402683822","https://openalex.org/W4402684150","https://openalex.org/W4402684217","https://openalex.org/W4402684345","https://openalex.org/W4404780875","https://openalex.org/W4404781033","https://openalex.org/W4404781196","https://openalex.org/W4404781197","https://openalex.org/W4404781211","https://openalex.org/W4404781412","https://openalex.org/W4404781582","https://openalex.org/W4404781628","https://openalex.org/W4404781825","https://openalex.org/W4404781959","https://openalex.org/W4404782102","https://openalex.org/W4404782508","https://openalex.org/W4404782624","https://openalex.org/W4404782735","https://openalex.org/W4404782781","https://openalex.org/W4404782868","https://openalex.org/W4404782923","https://openalex.org/W4404782935","https://openalex.org/W4404782947","https://openalex.org/W4404783024","https://openalex.org/W4404783074","https://openalex.org/W4404783248","https://openalex.org/W4404783391","https://openalex.org/W4404783465","https://openalex.org/W4404783757","https://openalex.org/W4404783767","https://openalex.org/W4404784125","https://openalex.org/W4404792814","https://openalex.org/W4404792842","https://openalex.org/W4404792846","https://openalex.org/W4406603970","https://openalex.org/W4406975393","https://openalex.org/W4412888822","https://openalex.org/W4413918681"],"related_works":[],"abstract_inverted_index":{"This":[0,150],"survey":[1],"examines":[2],"evaluation":[3,82,126,222],"methods":[4,147],"for":[5,39,77,199,246,258],"large":[6],"language":[7,159],"model":[8],"(LLM)-based":[9],"agents":[10,76,110],"in":[11,113,191],"multi-turn":[12,78,179,233],"conversational":[13,109,250],"settings.":[14],"Using":[15],"a":[16,36,45,114,196,255],"PRISMA-inspired":[17],"framework,":[18],"we":[19,208],"systematically":[20],"reviewed":[21],"nearly":[22],"250":[23],"scholarly":[24],"sources,":[25],"capturing":[26],"the":[27,106,125,174,185,203,217,244],"state-of-the-art":[28],"from":[29,158],"various":[30],"venues":[31],"of":[32,74,108,178,205,219],"publication,":[33],"and":[34,60,80,92,99,116,145,164,194,212,227,253,260],"establishing":[35],"solid":[37],"foundation":[38],"our":[40],"analysis.":[41],"Our":[42,235],"study":[43],"offers":[44],"structured":[46,197],"approach":[47],"by":[48],"developing":[49],"two":[50],"interrelated":[51],"taxonomy":[52,70,121],"systems:":[53],"one":[54],"that":[55,62,105,138,172,230],"defines":[56],"what":[57],"to":[58,65],"evaluate":[59,66],"another":[61],"explains":[63],"how":[64],".":[67],"The":[68,119],"first":[69],"identifies":[71],"key":[72],"components":[73,103],"LLM-based":[75],"conversations":[79],"their":[81],"dimensions,":[83],"including":[84,216],"task":[85],"completion,":[86],"response":[87],"quality,":[88],"user":[89],"experience,":[90],"memory":[91],"context":[93],"retention,":[94],"as":[95,97,162],"well":[96],"planning":[98],"tool":[100],"integration.":[101],"These":[102],"ensure":[104],"performance":[107],"is":[111],"assessed":[112],"holistic":[115],"meaningful":[117],"manner.":[118],"second":[120],"system":[122],"focuses":[123],"on":[124,202],"methodologies.":[127],"It":[128],"categorizes":[129],"approaches":[130],"into":[131],"annotation-based":[132],"evaluations,":[133],"automated":[134],"metrics,":[135],"hybrid":[136],"strategies":[137],"combine":[139],"human":[140],"assessments":[141],"with":[142,240],"quantitative":[143],"measures,":[144],"self-judging":[146],"utilizing":[148],"LLMs.":[149],"framework":[151],"not":[152],"only":[153],"captures":[154],"traditional":[155,192],"metrics":[156,229],"derived":[157],"understanding,":[160],"such":[161],"BLEU":[163],"ROUGE":[165],"scores,":[166],"but":[167],"also":[168],"incorporates":[169],"advanced":[170],"techniques":[171],"reflect":[173],"dynamic,":[175],"interactive":[176],"nature":[177],"dialogues.":[180],"Together,":[181],"these":[182],"frameworks":[183],"summarize":[184],"current":[186],"status":[187],"quo,":[188],"expose":[189],"limitations":[190],"practices,":[193,242],"provide":[195],"blueprint":[198],"improvement.":[200],"Based":[201],"summarization":[204],"existing":[206],"studies,":[207],"identify":[209],"several":[210],"challenges":[211],"propose":[213],"future":[214],"directions,":[215],"development":[218],"scalable,":[220],"real-time":[221],"pipelines,":[223],"enhanced":[224],"privacy-preserving":[225],"mechanisms,":[226],"robust":[228],"capture":[231],"dynamic":[232],"interactions.":[234],"contributions":[236],"bridge":[237],"historical":[238],"insights":[239],"modern":[241],"paving":[243],"way":[245],"next-generation,":[247],"reliably":[248],"evaluated":[249],"AI":[251],"systems":[252],"offering":[254],"comprehensive":[256],"guide":[257],"researchers":[259],"practitioners.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":3}],"updated_date":"2026-07-17T09:13:05.818461","created_date":"2026-02-15T00:00:00"}