{"id":"https://openalex.org/W3156287428","doi":"https://doi.org/10.1145/3404835.3463241","title":"Simulating User Satisfaction for the Evaluation of Task-oriented Dialogue Systems","display_name":"Simulating User Satisfaction for the Evaluation of Task-oriented Dialogue Systems","publication_year":2021,"publication_date":"2021-07-11","ids":{"openalex":"https://openalex.org/W3156287428","doi":"https://doi.org/10.1145/3404835.3463241","mag":"3156287428"},"language":"en","primary_location":{"id":"doi:10.1145/3404835.3463241","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463241","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2105.03748","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Weiwei Sun","orcid":null},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Weiwei Sun","raw_affiliation_strings":["Shandong University, Qingdao, China"],"affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Shuo Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shuo Zhang","raw_affiliation_strings":["Bloomberg, London, United Kingdom"],"affiliations":[{"raw_affiliation_string":"Bloomberg, London, United Kingdom","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Krisztian Balog","orcid":null},"institutions":[{"id":"https://openalex.org/I92008406","display_name":"University of Stavanger","ror":"https://ror.org/02qte9q33","country_code":"NO","type":"education","lineage":["https://openalex.org/I92008406"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Krisztian Balog","raw_affiliation_strings":["University of Stavanger, Stavanger, Norway"],"affiliations":[{"raw_affiliation_string":"University of Stavanger, Stavanger, Norway","institution_ids":["https://openalex.org/I92008406"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhaochun Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaochun Ren","raw_affiliation_strings":["Shandong University, Qingdao, China"],"affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Pengjie Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pengjie Ren","raw_affiliation_strings":["Shandong University, Qingdao, China"],"affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhumin Chen","orcid":null},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhumin Chen","raw_affiliation_strings":["Shandong University, Qingdao, China"],"affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"last","author":{"id":null,"display_name":"Maarten de Rijke","orcid":null},"institutions":[{"id":"https://openalex.org/I887064364","display_name":"University of Amsterdam","ror":"https://ror.org/04dkp9463","country_code":"NL","type":"education","lineage":["https://openalex.org/I887064364"]},{"id":"https://openalex.org/I4210112722","display_name":"Ahold Delhaize (Netherlands)","ror":"https://ror.org/01v6p2g18","country_code":"NL","type":"company","lineage":["https://openalex.org/I4210112722"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maarten de Rijke","raw_affiliation_strings":["University of Amsterdam & Ahold Delhaize Research, Amsterdam, Netherlands"],"affiliations":[{"raw_affiliation_string":"University of Amsterdam & Ahold Delhaize Research, Amsterdam, Netherlands","institution_ids":["https://openalex.org/I4210112722","https://openalex.org/I887064364"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I154099455"],"apc_list":null,"apc_paid":null,"fwci":3.3596,"has_fulltext":false,"cited_by_count":33,"citation_normalized_percentile":{"value":0.93350511,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":93,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"2499","last_page":"2506"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.6233999729156494},{"id":"https://openalex.org/keywords/user-satisfaction","display_name":"User satisfaction","score":0.6158000230789185},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5993000268936157},{"id":"https://openalex.org/keywords/computer-user-satisfaction","display_name":"Computer user satisfaction","score":0.5820000171661377},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5735999941825867},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.5519999861717224},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5041000247001648},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.46810001134872437}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8062000274658203},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.6233999729156494},{"id":"https://openalex.org/C3017893058","wikidata":"https://www.wikidata.org/wiki/Q999278","display_name":"User satisfaction","level":2,"score":0.6158000230789185},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5993000268936157},{"id":"https://openalex.org/C63880386","wikidata":"https://www.wikidata.org/wiki/Q5157592","display_name":"Computer user satisfaction","level":4,"score":0.5820000171661377},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5735999941825867},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.5519999861717224},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5041000247001648},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.49309998750686646},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.46810001134872437},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4465000033378601},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4339999854564667},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.4239000082015991},{"id":"https://openalex.org/C2776321320","wikidata":"https://www.wikidata.org/wiki/Q857525","display_name":"Annotation","level":2,"score":0.4237000048160553},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.38920000195503235},{"id":"https://openalex.org/C67712803","wikidata":"https://www.wikidata.org/wiki/Q7901853","display_name":"User modeling","level":3,"score":0.3806999921798706},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.3547999858856201},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.3528999984264374},{"id":"https://openalex.org/C143267671","wikidata":"https://www.wikidata.org/wiki/Q7901856","display_name":"User requirements document","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.322299987077713},{"id":"https://openalex.org/C91262260","wikidata":"https://www.wikidata.org/wiki/Q528074","display_name":"End user","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.28790000081062317},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2612000107765198},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.25949999690055847},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.251800000667572}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3404835.3463241","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3404835.3463241","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval","raw_type":"proceedings-article"},{"id":"pmh:oai:dare.uva.nl:openaire_cris_publications/b41a5f11-6137-4ab1-bebd-436be4f301ce","is_oa":false,"landing_page_url":"https://handle.uba.uva.nl/personal/pure/en/publications/simulating-user-satisfaction-for-the-evaluation-of-taskoriented-dialogue-systems(b41a5f11-6137-4ab1-bebd-436be4f301ce).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306400088","display_name":"UvA-DARE (University of Amsterdam)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I887064364","host_organization_name":"University of Amsterdam","host_organization_lineage":["https://openalex.org/I887064364"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Sun, W, Zhang, S, Balog, K, Ren, Z, Ren, P, Chen, Z & de Rijke, M 2021, Simulating User Satisfaction for the Evaluation of Task-oriented Dialogue Systems. in SIGIR '21 : proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval : July 11-15, 2021, virtual event, Canada. Association for Computing Machinery, New York, NY, pp. 2499-2506, 44th International ACM SIGIR Conference on Research and Development in Information Retrieval, Montr\u00e9al, Quebec, Canada, 11/07/21. https://doi.org/10.1145/3404835.3463241","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:arXiv.org:2105.03748","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.03748","pdf_url":"https://arxiv.org/pdf/2105.03748","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:uvapub:oai:dare.uva.nl:publications/b41a5f11-6137-4ab1-bebd-436be4f301ce","is_oa":false,"landing_page_url":"https://dare.uva.nl/personal/pure/en/publications/simulating-user-satisfaction-for-the-evaluation-of-taskoriented-dialogue-systems(b41a5f11-6137-4ab1-bebd-436be4f301ce).html","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"SIGIR '21: proceedings of the 44th International ACM SIGIR Conference on Research and Development in Information Retrieval : July 11-15, 2021, virtual event, Canada, 2499 - 2506","raw_type":"info:eu-repo/semantics/conferencepaper"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2105.03748","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2105.03748","pdf_url":"https://arxiv.org/pdf/2105.03748","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":21,"referenced_works":["https://openalex.org/W160067033","https://openalex.org/W969718702","https://openalex.org/W1586218930","https://openalex.org/W1975016129","https://openalex.org/W2062175565","https://openalex.org/W2113596624","https://openalex.org/W2121325257","https://openalex.org/W2128965063","https://openalex.org/W2470673105","https://openalex.org/W2563928368","https://openalex.org/W2604580630","https://openalex.org/W2889448364","https://openalex.org/W2944069152","https://openalex.org/W2962682659","https://openalex.org/W2962796276","https://openalex.org/W2962883855","https://openalex.org/W2972156721","https://openalex.org/W2997771882","https://openalex.org/W3034548376","https://openalex.org/W3034739704","https://openalex.org/W3155601106"],"related_works":[],"abstract_inverted_index":{"Evaluation":[0],"is":[1,71],"crucial":[2],"in":[3,127,182],"the":[4,46,53,60,69,74,83,133,162,179],"development":[5],"process":[6],"of":[7,48,62,68,77,91],"task-oriented":[8,63,114],"dialogue":[9,64],"systems.":[10,65],"As":[11],"an":[12],"evaluation":[13,61,75],"method,":[14],"user":[15,41,57,78,97,125,152,184],"simulation":[16,84],"allows":[17],"us":[18],"to":[19,72,81],"tackle":[20],"issues":[21],"such":[22],"as":[23,130,132],"scalability":[24],"and":[25,80,120,155],"cost-efficiency,":[26],"making":[27],"it":[28],"a":[29,39,49,89,96,141,188],"viable":[30],"choice":[31],"for":[32,59,151],"large-scale":[33],"automatic":[34],"evaluation.":[35],"To":[36,87],"help":[37],"build":[38],"human-like":[40],"simulator":[42],"that":[43,102,166],"can":[44],"measure":[45],"quality":[47],"dialogue,":[50],"we":[51,94],"propose":[52,95],"following":[54],"task:":[55],"simulating":[56],"satisfaction":[58,98,143,153,185],"The":[66],"purpose":[67],"task":[70],"increase":[73],"power":[76],"simulations":[79],"make":[82],"more":[85],"human-like.":[86],"overcome":[88],"lack":[90],"annotated":[92],"data,":[93],"annotation":[99],"dataset,":[100],"USS,":[101],"includes":[103],"6,800":[104],"dialogues":[105,115,134],"sampled":[106],"from":[107],"multiple":[108],"domains,":[109],"spanning":[110],"real-world":[111],"e-commerce":[112],"dialogues,":[113,129],"constructed":[116],"through":[117],"Wizard-of-Oz":[118],"experiments,":[119],"movie":[121],"recommendation":[122],"dialogues.":[123],"All":[124],"utterances":[126],"those":[128],"well":[131],"themselves,":[135],"have":[136],"been":[137],"labeled":[138],"based":[139,174],"on":[140,161,175],"5-level":[142],"scale.":[144],"We":[145],"also":[146],"share":[147],"three":[148],"baseline":[149],"methods":[150],"prediction":[154,157],"action":[156],"tasks.":[158],"Experiments":[159],"conducted":[160],"USS":[163],"dataset":[164],"suggest":[165],"distributed":[167],"representations":[168],"outperform":[169],"feature-based":[170],"methods.":[171],"A":[172],"model":[173,190],"hierarchical":[176],"GRUs":[177],"achieves":[178],"best":[180],"performance":[181],"in-domain":[183],"prediction,":[186],"while":[187],"BERT-based":[189],"has":[191],"better":[192],"cross-domain":[193],"generalization":[194],"ability.":[195]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2}],"updated_date":"2026-03-20T23:20:44.827607","created_date":"2021-04-26T00:00:00"}