{"id":"https://openalex.org/W4412876966","doi":"https://doi.org/10.1145/3711896.3737390","title":"A Framework for Evaluating AI Agents in Open-Ended Conversations via Scripted Simulation","display_name":"A Framework for Evaluating AI Agents in Open-Ended Conversations via Scripted Simulation","publication_year":2025,"publication_date":"2025-08-03","ids":{"openalex":"https://openalex.org/W4412876966","doi":"https://doi.org/10.1145/3711896.3737390"},"language":"en","primary_location":{"id":"doi:10.1145/3711896.3737390","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737390","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737390","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737390","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061732899","display_name":"Clarice Wang","orcid":"https://orcid.org/0009-0002-7589-4670"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Clarice Wang","raw_affiliation_strings":["University of Pennsylvania, Philadelphia, PA, USA"],"raw_orcid":"https://orcid.org/0009-0002-7589-4670","affiliations":[{"raw_affiliation_string":"University of Pennsylvania, Philadelphia, PA, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060349713","display_name":"Yimin Shi","orcid":"https://orcid.org/0000-0003-3375-4602"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yimin Shi","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-3375-4602","affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010903591","display_name":"Xiaokui Xiao","orcid":"https://orcid.org/0000-0003-0914-4580"},"institutions":[{"id":"https://openalex.org/I165932596","display_name":"National University of Singapore","ror":"https://ror.org/01tgyzw49","country_code":"SG","type":"education","lineage":["https://openalex.org/I165932596"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Xiaokui Xiao","raw_affiliation_strings":["National University of Singapore, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-0914-4580","affiliations":[{"raw_affiliation_string":"National University of Singapore, Singapore, Singapore","institution_ids":["https://openalex.org/I165932596"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.4183,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.92849461,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"5810","last_page":"5818"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.992900013923645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13643","display_name":"Artificial Intelligence in Law","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/3320","display_name":"Political Science and International Relations"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9679999947547913,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6813854575157166},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.45742687582969666}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6813854575157166},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.45742687582969666}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3711896.3737390","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737390","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737390","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3711896.3737390","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3711896.3737390","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3711896.3737390","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1795493338","display_name":null,"funder_award_id":"MOE-MOET32022-0001","funder_id":"https://openalex.org/F4320320751","funder_display_name":"Ministry of Education - Singapore"}],"funders":[{"id":"https://openalex.org/F4320320751","display_name":"Ministry of Education - Singapore","ror":"https://ror.org/01kcva023"},{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320323346","display_name":"B\u1ed9 Gi\u00e1o d\u1ee5c v\u00e0 \u00d0\u00e0o t\u1ea1o","ror":"https://ror.org/00drv3378"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4412876966.pdf","grobid_xml":"https://content.openalex.org/works/W4412876966.grobid-xml"},"referenced_works_count":22,"referenced_works":["https://openalex.org/W1510393856","https://openalex.org/W2011799942","https://openalex.org/W2063866393","https://openalex.org/W2158962793","https://openalex.org/W2794325560","https://openalex.org/W3083410900","https://openalex.org/W4221143046","https://openalex.org/W4223908421","https://openalex.org/W4226278401","https://openalex.org/W4367000491","https://openalex.org/W4377297670","https://openalex.org/W4378189609","https://openalex.org/W4380353763","https://openalex.org/W4385327683","https://openalex.org/W4386908092","https://openalex.org/W4387561453","https://openalex.org/W4392679398","https://openalex.org/W4404782588","https://openalex.org/W6849590751","https://openalex.org/W6852449896","https://openalex.org/W6856086692","https://openalex.org/W6856697067"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Traditional":[0],"evaluations":[1],"of":[2],"conversational":[3,176],"AI":[4,17,44,58,84,94,129,186],"have":[5],"primarily":[6],"focused":[7],"on":[8],"\"closed-ended\"":[9],"interactions,":[10],"where":[11],"a":[12,78,92,99,105,171],"human":[13,55,100],"user":[14],"queries":[15],"the":[16,43,54,127,139,146,151],"system,":[18],"such":[19,87,160],"as":[20,28,161],"in":[21,41,69,86,134,194],"customer":[22],"support.However,":[23],"many":[24],"advanced":[25],"real-world":[26],"applications-such":[27],"job":[29],"interviews,":[30],"podcast":[31],"hosting,":[32],"and":[33,112,143,164,190],"legal":[34],"or":[35,122],"healthcare":[36],"intake":[37],"discussions-require":[38],"\"open-ended\"":[39],"interactions":[40],"which":[42],"must":[45],"take":[46],"initiative":[47],"by":[48,102],"formulating":[49],"questions":[50],"to":[51,119,137,150],"fully":[52],"understand":[53],"user's":[56],"story.As":[57],"assumes":[59],"broader":[60],"roles":[61],"that":[62,108],"demand":[63],"greater":[64],"autonomy,":[65],"evaluating":[66],"its":[67],"performance":[68],"open-ended":[70,88,175],"conversations":[71],"becomes":[72],"significantly":[73],"more":[74],"complex.This":[75],"paper":[76],"introduces":[77],"novel":[79],"framework":[80],"for":[81,117,174],"rigorously":[82],"assessing":[83],"agents":[85],"interviews.In":[89],"this":[90],"framework,":[91],"secondary":[93],"agent":[95,130,187],"(Agent":[96,131],"B)":[97],"simulates":[98],"interviewee":[101],"strictly":[103],"following":[104],"structured":[106],"script":[107],"defines":[109],"topic":[110],"strengths":[111],"weaknesses,":[113],"along":[114],"with":[115],"guidelines":[116],"when":[118],"hint":[120],"at":[121],"deviate":[123],"from":[124],"these":[125],"topics.Meanwhile,":[126],"primary":[128],"A)":[132],"engages":[133],"dynamic":[135],"questioning":[136],"uncover":[138],"script's":[140],"underlying":[141],"facts":[142],"narrative.By":[144],"comparing":[145],"final":[147],"conversation":[148],"transcript":[149],"original":[152],"script,":[153],"we":[154],"assess":[155],"Agent":[156],"A":[157],"using":[158],"metrics":[159],"completeness,":[162],"consistency,":[163],"investigative":[165],"depth.This":[166],"approach":[167],"not":[168],"only":[169],"establishes":[170],"new":[172],"benchmark":[173],"skills":[177],"but":[178],"also":[179],"provides":[180],"insight":[181],"into":[182],"how":[183],"effectively":[184],"an":[185],"can":[188],"detect":[189],"navigate":[191],"strategic":[192],"diversions":[193],"scripted":[195],"behavior.":[196]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-06-17T08:01:34.144755","created_date":"2025-10-10T00:00:00"}