{"id":"https://openalex.org/W2109038907","doi":"https://doi.org/10.1017/s0269888912000343","title":"A survey on metrics for the evaluation of user simulations","display_name":"A survey on metrics for the evaluation of user simulations","publication_year":2012,"publication_date":"2012-11-28","ids":{"openalex":"https://openalex.org/W2109038907","doi":"https://doi.org/10.1017/s0269888912000343","mag":"2109038907"},"language":"en","primary_location":{"id":"doi:10.1017/s0269888912000343","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0269888912000343","pdf_url":null,"source":{"id":"https://openalex.org/S137506714","display_name":"The Knowledge Engineering Review","issn_l":"0269-8889","issn":["0269-8889","1469-8005"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Knowledge Engineering Review","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065100569","display_name":"Olivier Pietquin","orcid":"https://orcid.org/0000-0002-5386-465X"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"funder","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210089931","display_name":"Georgia Tech-CNRS Laboratory","ror":"https://ror.org/00avmbz91","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1288283403","https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I130701444","https://openalex.org/I4210089931","https://openalex.org/I4210095849","https://openalex.org/I4407990293","https://openalex.org/I90183372"]},{"id":"https://openalex.org/I4210160189","display_name":"Institut Polytechnique de Bordeaux","ror":"https://ror.org/054qv7y42","country_code":"FR","type":"education","lineage":["https://openalex.org/I4210160189"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Olivier Pietquin","raw_affiliation_strings":["SUPELEC \u2013 IMS-MaLIS Research Group, UMI 2958 (GeorgiaTech \u2013 CNRS), 2 rue Edouard Belin, 57070 Metz, France; e-mail:","e-mail:","SUPELEC \u2013 IMS-MaLIS Research Group, UMI 2958 (GeorgiaTech \u2013 CNRS), 2 rue Edouard Belin, 57070 Metz, France"],"affiliations":[{"raw_affiliation_string":"SUPELEC \u2013 IMS-MaLIS Research Group, UMI 2958 (GeorgiaTech \u2013 CNRS), 2 rue Edouard Belin, 57070 Metz, France; e-mail:","institution_ids":["https://openalex.org/I4210089931","https://openalex.org/I4210160189","https://openalex.org/I1294671590"]},{"raw_affiliation_string":"e-mail:","institution_ids":[]},{"raw_affiliation_string":"SUPELEC \u2013 IMS-MaLIS Research Group, UMI 2958 (GeorgiaTech \u2013 CNRS), 2 rue Edouard Belin, 57070 Metz, France","institution_ids":["https://openalex.org/I4210160189","https://openalex.org/I1294671590"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5084819987","display_name":"Helen Hastie","orcid":"https://orcid.org/0000-0002-9177-7282"},"institutions":[{"id":"https://openalex.org/I32062511","display_name":"Heriot-Watt University","ror":"https://ror.org/04mghma93","country_code":"GB","type":"education","lineage":["https://openalex.org/I32062511"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Helen Hastie","raw_affiliation_strings":["School of Mathematical and Computer Sciences, Heriot-Watt University, Edinburgh EH14 4AS, UK; e-mail:","School of Mathematical and Computer Sciences, Heriot-Watt University, Edinburgh EH14 4AS, UK","e-mail:"],"affiliations":[{"raw_affiliation_string":"School of Mathematical and Computer Sciences, Heriot-Watt University, Edinburgh EH14 4AS, UK; e-mail:","institution_ids":["https://openalex.org/I32062511"]},{"raw_affiliation_string":"School of Mathematical and Computer Sciences, Heriot-Watt University, Edinburgh EH14 4AS, UK","institution_ids":["https://openalex.org/I32062511"]},{"raw_affiliation_string":"e-mail:","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5065100569"],"corresponding_institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210089931","https://openalex.org/I4210160189"],"apc_list":null,"apc_paid":null,"fwci":10.3172,"has_fulltext":false,"cited_by_count":94,"citation_normalized_percentile":{"value":0.98183031,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":"28","issue":"1","first_page":"59","last_page":"73"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.875617265701294},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7414880990982056},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.6253467202186584},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5944119095802307},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5493766665458679},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.48256149888038635},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.4438480734825134},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44294169545173645}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.875617265701294},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7414880990982056},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.6253467202186584},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5944119095802307},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5493766665458679},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.48256149888038635},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.4438480734825134},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44294169545173645},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1017/s0269888912000343","is_oa":false,"landing_page_url":"https://doi.org/10.1017/s0269888912000343","pdf_url":null,"source":{"id":"https://openalex.org/S137506714","display_name":"The Knowledge Engineering Review","issn_l":"0269-8889","issn":["0269-8889","1469-8005"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"The Knowledge Engineering Review","raw_type":"journal-article"},{"id":"pmh:oai:HAL:hal-00771654v1","is_oa":false,"landing_page_url":"https://centralesupelec.hal.science/hal-00771654","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Knowledge Engineering Review, 2013, 28 (1), pp.59-73. ⟨10.1017/S0269888912000343⟩","raw_type":"Journal articles"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W1375381","https://openalex.org/W62710299","https://openalex.org/W91852349","https://openalex.org/W160067033","https://openalex.org/W343799206","https://openalex.org/W1584761227","https://openalex.org/W1600735390","https://openalex.org/W1851612831","https://openalex.org/W1875550513","https://openalex.org/W1965555277","https://openalex.org/W1987725259","https://openalex.org/W1996957559","https://openalex.org/W1997319212","https://openalex.org/W2025989261","https://openalex.org/W2031571562","https://openalex.org/W2061562262","https://openalex.org/W2062175565","https://openalex.org/W2078861931","https://openalex.org/W2089327077","https://openalex.org/W2095977914","https://openalex.org/W2101105183","https://openalex.org/W2106547558","https://openalex.org/W2112297877","https://openalex.org/W2116154161","https://openalex.org/W2117989772","https://openalex.org/W2121360982","https://openalex.org/W2121863487","https://openalex.org/W2128965063","https://openalex.org/W2132997613","https://openalex.org/W2134051188","https://openalex.org/W2134466368","https://openalex.org/W2151814822","https://openalex.org/W2153804780","https://openalex.org/W2155898337","https://openalex.org/W2162265730","https://openalex.org/W2163068732","https://openalex.org/W2166445349","https://openalex.org/W2168490009","https://openalex.org/W2171079152","https://openalex.org/W2171804848","https://openalex.org/W2324707440","https://openalex.org/W2396863940","https://openalex.org/W2438667436","https://openalex.org/W2603612888","https://openalex.org/W2624335110","https://openalex.org/W2757979647","https://openalex.org/W2950502481","https://openalex.org/W4214717370","https://openalex.org/W4243308763","https://openalex.org/W4285719527","https://openalex.org/W4323322863","https://openalex.org/W6603742574","https://openalex.org/W6682780409","https://openalex.org/W6683794156"],"related_works":["https://openalex.org/W4306904969","https://openalex.org/W2138720691","https://openalex.org/W4362501864","https://openalex.org/W4380318855","https://openalex.org/W2031695474","https://openalex.org/W2024136090","https://openalex.org/W2586732548","https://openalex.org/W3049728571","https://openalex.org/W2964765435","https://openalex.org/W4391331176"],"abstract_inverted_index":{"Abstract":[0],"User":[1,41,135],"simulation":[2,78,109],"is":[3,23,68,79,111,122],"an":[4,112],"important":[5],"research":[6],"area":[7],"in":[8,90,138,148],"the":[9,61,76,88,97,102,139],"field":[10],"of":[11,63,75,81,92,101,104,134,150,153],"spoken":[12],"dialogue":[13,39,53],"systems":[14],"(SDSs)":[15],"because":[16,84],"collecting":[17],"and":[18,26,37,96,107,144],"annotating":[19],"real":[20],"human\u2013machine":[21],"interactions":[22],"often":[24],"expensive":[25],"time-consuming.":[27],"However,":[28],"such":[29,56],"data":[30,64],"are":[31,43,119],"generally":[32],"required":[33],"for":[34,51,66],"designing,":[35],"training":[36,67],"assessing":[38],"systems.":[40],"simulations":[42],"especially":[44],"needed":[45],"when":[46],"using":[47],"machine":[48],"learning":[49],"methods":[50,110],"optimizing":[52],"management":[54],"strategies":[55],"as":[57],"Reinforcement":[58],"Learning,":[59],"where":[60],"amount":[62],"necessary":[65],"larger":[69],"than":[70],"existing":[71],"corpora.":[72],"The":[73],"quality":[74,103],"user":[77,108],"therefore":[80],"crucial":[82],"importance":[83],"it":[85],"dramatically":[86],"influences":[87],"results":[89],"terms":[91,149],"SDS":[93],"performance":[94],"analysis":[95],"learnt":[98],"strategy.":[99],"Assessment":[100],"simulated":[105],"dialogues":[106],"open":[113],"issue":[114],"and,":[115],"although":[116],"assessment":[117],"metrics":[118,147],"required,":[120],"there":[121],"no":[123],"commonly":[124],"adopted":[125],"metric.":[126],"In":[127],"this":[128],"paper,":[129],"we":[130],"give":[131],"a":[132,151],"survey":[133],"Simulations":[136],"Metrics":[137],"literature,":[140],"propose":[141],"some":[142],"extensions":[143],"discuss":[145],"these":[146],"list":[152],"desired":[154],"features.":[155]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":8},{"year":2019,"cited_by_count":9},{"year":2018,"cited_by_count":13},{"year":2017,"cited_by_count":12},{"year":2016,"cited_by_count":4},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":5},{"year":2013,"cited_by_count":7},{"year":2012,"cited_by_count":8}],"updated_date":"2026-04-04T08:04:53.788161","created_date":"2025-10-10T00:00:00"}