{"id":"https://openalex.org/W4396843659","doi":"https://doi.org/10.1145/3589335.3651955","title":"How Reliable is Your Simulator? Analysis on the Limitations of Current LLM-based User Simulators for Conversational Recommendation","display_name":"How Reliable is Your Simulator? Analysis on the Limitations of Current LLM-based User Simulators for Conversational Recommendation","publication_year":2024,"publication_date":"2024-05-12","ids":{"openalex":"https://openalex.org/W4396843659","doi":"https://doi.org/10.1145/3589335.3651955"},"language":"en","primary_location":{"id":"doi:10.1145/3589335.3651955","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3589335.3651955","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3589335.3651955","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2024","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3589335.3651955","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006694585","display_name":"Lixi Zhu","orcid":"https://orcid.org/0000-0002-7121-7771"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Lixi Zhu","raw_affiliation_strings":["School of Computer Science and Technology, Beijing Jiaotong University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076843247","display_name":"Xiaowen Huang","orcid":"https://orcid.org/0000-0001-9590-3285"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaowen Huang","raw_affiliation_strings":["School of Computer Science and Technology, Beijing Jiaotong University & Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Jiaotong University & Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijing, China","institution_ids":["https://openalex.org/I21193070"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023834030","display_name":"Jitao Sang","orcid":"https://orcid.org/0000-0002-0699-3205"},"institutions":[{"id":"https://openalex.org/I21193070","display_name":"Beijing Jiaotong University","ror":"https://ror.org/01yj56c84","country_code":"CN","type":"education","lineage":["https://openalex.org/I21193070"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jitao Sang","raw_affiliation_strings":["School of Computer Science and Technology, Beijing Jiaotong University & Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijng, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Jiaotong University & Beijing Key Lab of Traffic Data Analysis and Mining, Beijing Jiaotong University, Beijng, China","institution_ids":["https://openalex.org/I21193070"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5006694585"],"corresponding_institution_ids":["https://openalex.org/I21193070"],"apc_list":null,"apc_paid":null,"fwci":6.313,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.9637432,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"1726","last_page":"1732"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12384","display_name":"Customer churn and segmentation","score":0.9581000208854675,"subfield":{"id":"https://openalex.org/subfields/1406","display_name":"Marketing"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9496999979019165,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8038222789764404},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.53645920753479},{"id":"https://openalex.org/keywords/simulation","display_name":"Simulation","score":0.4414079785346985}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8038222789764404},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.53645920753479},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.4414079785346985}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3589335.3651955","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3589335.3651955","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3589335.3651955","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2024","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3589335.3651955","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3589335.3651955","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3589335.3651955","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Companion Proceedings of the ACM Web Conference 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4396843659.pdf"},"referenced_works_count":16,"referenced_works":["https://openalex.org/W2054141820","https://openalex.org/W2605350416","https://openalex.org/W2950457956","https://openalex.org/W2964112275","https://openalex.org/W2997662139","https://openalex.org/W3038439974","https://openalex.org/W3080122044","https://openalex.org/W3099865390","https://openalex.org/W3100790518","https://openalex.org/W3152509363","https://openalex.org/W3185784178","https://openalex.org/W4283324387","https://openalex.org/W4285066176","https://openalex.org/W4368755500","https://openalex.org/W4386081001","https://openalex.org/W6754706484"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W2382290278","https://openalex.org/W4395014643"],"abstract_inverted_index":{"Conversational":[0],"Recommender":[1],"System":[2],"(CRS)":[3],"interacts":[4],"with":[5,78,136],"users":[6],"through":[7,196],"natural":[8],"language":[9],"to":[10,27,63,89,102,214,230],"understand":[11],"their":[12],"preferences":[13],"and":[14,34,155,176],"provide":[15],"personalized":[16],"recommendations":[17,170],"in":[18,56,96,126,152,161],"real-time.":[19],"CRS":[20,169,228],"has":[21],"demonstrated":[22],"significant":[23],"potential,":[24],"prompting":[25],"researchers":[26],"address":[28],"the":[29,43,91,115,127,137,156,174,183,190,193,216,219,225,232,237],"development":[30],"of":[31,45,54,93,129,168,178,192,227],"more":[32,172],"realistic":[33],"reliable":[35],"user":[36,65,98,142,157,186,194],"simulators":[37,66,99,143],"as":[38],"a":[39,52,197,211],"key":[40],"focus.":[41],"Recently,":[42],"capabilities":[44],"Large":[46],"Language":[47],"Models":[48],"(LLMs)":[49],"have":[50],"attracted":[51],"lot":[53],"attention":[55],"various":[57],"fields.":[58],"Simultaneously,":[59],"efforts":[60],"are":[61],"underway":[62],"construct":[64],"based":[67,144],"on":[68,114,122,145,173,182],"LLMs.":[69],"While":[70],"these":[71,205],"works":[72],"showcase":[73],"innovation,":[74],"they":[75],"also":[76],"come":[77],"certain":[79],"limitations":[80,92],"that":[81],"require":[82],"attention.":[83],"In":[84],"this":[85,108],"work,":[86,117],"we":[87,110,132,207],"aim":[88],"analyze":[90],"using":[94],"LLMs":[95],"constructing":[97],"for":[100,141],"CRS,":[101],"guide":[103,215],"future":[104],"research.":[105],"To":[106,203],"achieve":[107],"goal,":[109],"conduct":[111],"analytical":[112],"validation":[113],"notable":[116],"iEvaLM.":[118],"Through":[119],"multiple":[120],"experiments":[121],"two":[123],"widely-used":[124],"datasets":[125],"field":[128],"conversational":[130,153,179],"recommendation,":[131],"highlight":[133],"several":[134],"issues":[135],"current":[138],"evaluation":[139,163],"methods":[140],"LLMs:":[146],"(1)":[147],"Data":[148],"leakage,":[149],"which":[150],"occurs":[151],"history":[154,180],"simulator's":[158],"replies,":[159],"results":[160],"inflated":[162],"results.":[164,239],"(2)":[165],"The":[166],"success":[167],"depends":[171],"availability":[175],"quality":[177],"than":[181],"responses":[184],"from":[185],"simulators.":[187],"(3)":[188],"Controlling":[189],"output":[191],"simulator":[195],"single":[198],"prompt":[199],"template":[200],"proves":[201],"challenging.":[202],"overcome":[204],"limitations,":[206],"propose":[208],"SimpleUserSim,":[209],"employing":[210],"straightforward":[212],"strategy":[213],"topic":[217],"toward":[218],"target":[220],"items.":[221],"Our":[222],"study":[223],"validates":[224],"ability":[226],"models":[229],"utilize":[231],"interaction":[233],"information,":[234],"significantly":[235],"improving":[236],"recommendation":[238]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":6}],"updated_date":"2026-03-24T08:02:53.985720","created_date":"2025-10-10T00:00:00"}