{"id":"https://openalex.org/W2944069152","doi":"https://doi.org/10.1007/s10462-020-09866-x","title":"Survey on evaluation methods for dialogue systems","display_name":"Survey on evaluation methods for dialogue systems","publication_year":2020,"publication_date":"2020-06-25","ids":{"openalex":"https://openalex.org/W2944069152","doi":"https://doi.org/10.1007/s10462-020-09866-x","mag":"2944069152"},"language":"en","primary_location":{"id":"doi:10.1007/s10462-020-09866-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10462-020-09866-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10462-020-09866-x.pdf","source":{"id":"https://openalex.org/S122814990","display_name":"Artificial Intelligence Review","issn_l":"0269-2821","issn":["0269-2821","1573-7462"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence Review","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10462-020-09866-x.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Jan Deriu","orcid":"https://orcid.org/0000-0002-8405-1344"},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Jan Deriu","raw_affiliation_strings":["Zurich University of Applied Sciences (ZHAW), Steinberggasse 13, 8400, Winterthur, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-8405-1344","affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences (ZHAW), Steinberggasse 13, 8400, Winterthur, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Alvaro Rodrigo","orcid":null},"institutions":[{"id":"https://openalex.org/I178450904","display_name":"Universidad Nacional de Educaci\u00f3n a Distancia","ror":"https://ror.org/02msb5n36","country_code":"ES","type":"education","lineage":["https://openalex.org/I178450904"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Alvaro Rodrigo","raw_affiliation_strings":["NLP & IRGroup, UNED, C/Juan del Rosal 16, 28040, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP & IRGroup, UNED, C/Juan del Rosal 16, 28040, Madrid, Spain","institution_ids":["https://openalex.org/I178450904"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Arantxa Otegi","orcid":null},"institutions":[{"id":"https://openalex.org/I169108374","display_name":"University of the Basque Country","ror":"https://ror.org/000xsnr85","country_code":"ES","type":"education","lineage":["https://openalex.org/I169108374"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Arantxa Otegi","raw_affiliation_strings":["IXA NLP Group, University of the Basque Country(UPV/EHU), Manuel Lardizabal 1 Donostia, 20018, Basque Country, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IXA NLP Group, University of the Basque Country(UPV/EHU), Manuel Lardizabal 1 Donostia, 20018, Basque Country, Spain","institution_ids":["https://openalex.org/I169108374"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Guillermo Echegoyen","orcid":null},"institutions":[{"id":"https://openalex.org/I178450904","display_name":"Universidad Nacional de Educaci\u00f3n a Distancia","ror":"https://ror.org/02msb5n36","country_code":"ES","type":"education","lineage":["https://openalex.org/I178450904"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Guillermo Echegoyen","raw_affiliation_strings":["NLP & IRGroup, UNED, C/Juan del Rosal 16, 28040, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"NLP & IRGroup, UNED, C/Juan del Rosal 16, 28040, Madrid, Spain","institution_ids":["https://openalex.org/I178450904"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Sophie Rosset","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I277688954","display_name":"Universit\u00e9 Paris-Saclay","ror":"https://ror.org/03xjwb503","country_code":"FR","type":"education","lineage":["https://openalex.org/I277688954"]},{"id":"https://openalex.org/I4210115485","display_name":"Laboratoire d'Informatique pour la M\u00e9canique et les Sciences de l'Ing\u00e9nieur","ror":"https://ror.org/01raq4x89","country_code":"FR","type":"facility","lineage":["https://openalex.org/I102197404","https://openalex.org/I1294671590","https://openalex.org/I4210115485","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Sophie Rosset","raw_affiliation_strings":["CNRS, LIMSI, Universit\u00e9 Paris-Saclay, Campus Universitaire, B\u00e2t. 508, rue John von Neumann, 91405, Orsay Cedex, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CNRS, LIMSI, Universit\u00e9 Paris-Saclay, Campus Universitaire, B\u00e2t. 508, rue John von Neumann, 91405, Orsay Cedex, France","institution_ids":["https://openalex.org/I277688954","https://openalex.org/I4210115485","https://openalex.org/I1294671590"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Eneko Agirre","orcid":null},"institutions":[{"id":"https://openalex.org/I169108374","display_name":"University of the Basque Country","ror":"https://ror.org/000xsnr85","country_code":"ES","type":"education","lineage":["https://openalex.org/I169108374"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Eneko Agirre","raw_affiliation_strings":["IXA NLP Group, University of the Basque Country(UPV/EHU), Manuel Lardizabal 1 Donostia, 20018, Basque Country, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IXA NLP Group, University of the Basque Country(UPV/EHU), Manuel Lardizabal 1 Donostia, 20018, Basque Country, Spain","institution_ids":["https://openalex.org/I169108374"]}]},{"author_position":"last","author":{"id":null,"display_name":"Mark Cieliebak","orcid":null},"institutions":[{"id":"https://openalex.org/I858936495","display_name":"ZHAW Zurich University of Applied Sciences","ror":"https://ror.org/05pmsvm27","country_code":"CH","type":"education","lineage":["https://openalex.org/I858936495"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Mark Cieliebak","raw_affiliation_strings":["Zurich University of Applied Sciences (ZHAW), Steinberggasse 13, 8400, Winterthur, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zurich University of Applied Sciences (ZHAW), Steinberggasse 13, 8400, Winterthur, Switzerland","institution_ids":["https://openalex.org/I858936495"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I858936495"],"apc_list":{"value":2490,"currency":"EUR","value_usd":3090},"apc_paid":{"value":2490,"currency":"EUR","value_usd":3090},"fwci":12.1042,"has_fulltext":true,"cited_by_count":187,"citation_normalized_percentile":{"value":0.98881419,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"54","issue":"1","first_page":"755","last_page":"810"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7921000123023987,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.7921000123023987,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.042399998754262924,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.016899999231100082,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/cover","display_name":"Cover (algebra)","score":0.5755000114440918},{"id":"https://openalex.org/keywords/evaluation-methods","display_name":"Evaluation methods","score":0.5609999895095825},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.5033000111579895},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.4284000098705292},{"id":"https://openalex.org/keywords/reduction","display_name":"Reduction (mathematics)","score":0.3231000006198883}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7985000014305115},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.5755000114440918},{"id":"https://openalex.org/C3018395757","wikidata":"https://www.wikidata.org/wiki/Q1379672","display_name":"Evaluation methods","level":2,"score":0.5609999895095825},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.5033000111579895},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.4284000098705292},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.3831000030040741},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3668000102043152},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3531000018119812},{"id":"https://openalex.org/C56739046","wikidata":"https://www.wikidata.org/wiki/Q192060","display_name":"Knowledge management","level":1,"score":0.3391000032424927},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.3231000006198883},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.29179999232292175},{"id":"https://openalex.org/C180198813","wikidata":"https://www.wikidata.org/wiki/Q121182","display_name":"Information system","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C5395021","wikidata":"https://www.wikidata.org/wiki/Q1061410","display_name":"Program evaluation","level":2,"score":0.2531000077724457}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1007/s10462-020-09866-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10462-020-09866-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10462-020-09866-x.pdf","source":{"id":"https://openalex.org/S122814990","display_name":"Artificial Intelligence Review","issn_l":"0269-2821","issn":["0269-2821","1573-7462"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence Review","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1905.04071","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1905.04071","pdf_url":"https://arxiv.org/pdf/1905.04071","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-03006231v1","is_oa":true,"landing_page_url":"https://hal.science/hal-03006231","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Artificial Intelligence Review, 2020, 54 (1), pp.755-810. ⟨10.1007/s10462-020-09866-x⟩","raw_type":"Journal articles"},{"id":"pmh:oai:digitalcollection.zhaw.ch:11475/20318","is_oa":true,"landing_page_url":"https://hdl.handle.net/11475/20318","pdf_url":null,"source":{"id":"https://openalex.org/S4306401810","display_name":"Z\u00fcrcher Hochschule f\u00fcr Angewandte Wissenschaften digital collection (Zurich University of Applied Sciences)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200744771","host_organization_name":"ZHAW Zurich University of Applied Sciences","host_organization_lineage":["https://openalex.org/I200744771"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Text"},{"id":"pmh:oai:pubmedcentral.nih.gov:7817575","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7817575","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Artif Intell Rev","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.1007/s10462-020-09866-x","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10462-020-09866-x","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10462-020-09866-x.pdf","source":{"id":"https://openalex.org/S122814990","display_name":"Artificial Intelligence Review","issn_l":"0269-2821","issn":["0269-2821","1573-7462"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Artificial Intelligence Review","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G177785663","display_name":null,"funder_award_id":"PCIN-2017-118/AEI","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G195474297","display_name":null,"funder_award_id":"2017-","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G1978800246","display_name":null,"funder_award_id":"20CH21_174237","funder_id":"https://openalex.org/F4320320924","funder_display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung"},{"id":"https://openalex.org/G2653555252","display_name":null,"funder_award_id":"PCIN-2017-085/AEI","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G3242539578","display_name":null,"funder_award_id":"ANR-17-CHR2-0001-03","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G6073587987","display_name":"Learning to Interact with Humans by Lifelong Interaction with Humans","funder_award_id":"ANR-17-CHR2-0001","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G6803667113","display_name":null,"funder_award_id":"ANR-17","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"},{"id":"https://openalex.org/F4320321919","display_name":"Euskal Herriko Unibertsitatea","ror":"https://ror.org/000xsnr85"},{"id":"https://openalex.org/F4320335598","display_name":"Agencia Estatal de Investigaci\u00f3n","ror":null},{"id":"https://openalex.org/F4320338463","display_name":"CHIST-ERA","ror":"https://ror.org/00rbzpz17"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2944069152.pdf","grobid_xml":"https://content.openalex.org/works/W2944069152.grobid-xml"},"referenced_works_count":136,"referenced_works":["https://openalex.org/W119047706","https://openalex.org/W204133468","https://openalex.org/W225503657","https://openalex.org/W635530177","https://openalex.org/W969718702","https://openalex.org/W1482666487","https://openalex.org/W1518951372","https://openalex.org/W1535960497","https://openalex.org/W1586218930","https://openalex.org/W1649407914","https://openalex.org/W1948566616","https://openalex.org/W1971034924","https://openalex.org/W1975016129","https://openalex.org/W1975244201","https://openalex.org/W1975879668","https://openalex.org/W1976400535","https://openalex.org/W1976538119","https://openalex.org/W1979299372","https://openalex.org/W1986199072","https://openalex.org/W1990508473","https://openalex.org/W1993567041","https://openalex.org/W2001771035","https://openalex.org/W2024632416","https://openalex.org/W2035934535","https://openalex.org/W2047382442","https://openalex.org/W2056776201","https://openalex.org/W2057919624","https://openalex.org/W2059857707","https://openalex.org/W2062175565","https://openalex.org/W2064675550","https://openalex.org/W2070150502","https://openalex.org/W2080887205","https://openalex.org/W2098506269","https://openalex.org/W2101105183","https://openalex.org/W2109038907","https://openalex.org/W2113596624","https://openalex.org/W2115101920","https://openalex.org/W2129405869","https://openalex.org/W2134051188","https://openalex.org/W2137871902","https://openalex.org/W2139079654","https://openalex.org/W2139196777","https://openalex.org/W2140054881","https://openalex.org/W2146311236","https://openalex.org/W2154740693","https://openalex.org/W2156974606","https://openalex.org/W2157701451","https://openalex.org/W2166637769","https://openalex.org/W2168332560","https://openalex.org/W2168490009","https://openalex.org/W2169661988","https://openalex.org/W2170872814","https://openalex.org/W2183366530","https://openalex.org/W2250262342","https://openalex.org/W2251058040","https://openalex.org/W2251818205","https://openalex.org/W2280612018","https://openalex.org/W2291723583","https://openalex.org/W2361488420","https://openalex.org/W2418993857","https://openalex.org/W2429300145","https://openalex.org/W2468710617","https://openalex.org/W2476025067","https://openalex.org/W2482325199","https://openalex.org/W2489487449","https://openalex.org/W2521114121","https://openalex.org/W2557764419","https://openalex.org/W2565085003","https://openalex.org/W2580343853","https://openalex.org/W2584220694","https://openalex.org/W2586756136","https://openalex.org/W2593751037","https://openalex.org/W2594902929","https://openalex.org/W2606964149","https://openalex.org/W2612228435","https://openalex.org/W2626699878","https://openalex.org/W2739936944","https://openalex.org/W2759200442","https://openalex.org/W2786472750","https://openalex.org/W2786472963","https://openalex.org/W2788204336","https://openalex.org/W2794736157","https://openalex.org/W2799195065","https://openalex.org/W2810821963","https://openalex.org/W2885721522","https://openalex.org/W2886344013","https://openalex.org/W2904443424","https://openalex.org/W2914397182","https://openalex.org/W2923014074","https://openalex.org/W2962717182","https://openalex.org/W2962767298","https://openalex.org/W2962776342","https://openalex.org/W2962854379","https://openalex.org/W2962883855","https://openalex.org/W2962905474","https://openalex.org/W2962974452","https://openalex.org/W2963167310","https://openalex.org/W2963206148","https://openalex.org/W2963223306","https://openalex.org/W2963323070","https://openalex.org/W2963339397","https://openalex.org/W2963341956","https://openalex.org/W2963491014","https://openalex.org/W2963527228","https://openalex.org/W2963567240","https://openalex.org/W2963748441","https://openalex.org/W2963797754","https://openalex.org/W2963857397","https://openalex.org/W2963903950","https://openalex.org/W2963912046","https://openalex.org/W2963963856","https://openalex.org/W2963963993","https://openalex.org/W2964039645","https://openalex.org/W2964101860","https://openalex.org/W2964120615","https://openalex.org/W2964223283","https://openalex.org/W2969893967","https://openalex.org/W2970377674","https://openalex.org/W2973054254","https://openalex.org/W2979702391","https://openalex.org/W2985067290","https://openalex.org/W2985891764","https://openalex.org/W2986193249","https://openalex.org/W2987831083","https://openalex.org/W2997771882","https://openalex.org/W2998099211","https://openalex.org/W3022187094","https://openalex.org/W3118485687","https://openalex.org/W4235073904","https://openalex.org/W4236521339","https://openalex.org/W4247250990","https://openalex.org/W4254836149","https://openalex.org/W4290742115","https://openalex.org/W6600336842","https://openalex.org/W6600424091","https://openalex.org/W6823602480"],"related_works":["https://openalex.org/W2081647779","https://openalex.org/W3107474891","https://openalex.org/W1607472309","https://openalex.org/W3217387898","https://openalex.org/W3128051602","https://openalex.org/W4237750775","https://openalex.org/W3118295188","https://openalex.org/W2166831867","https://openalex.org/W2032348883","https://openalex.org/W1555670490"],"abstract_inverted_index":{"Abstract":[0],"In":[1,69],"this":[2,43,70],"paper,":[3],"we":[4,72,81],"survey":[5],"the":[6,12,27,74,84,102,107,113],"methods":[7,59,115],"and":[8,19,40,49,77,92,110],"concepts":[9,76],"developed":[10,105],"for":[11,106],"evaluation":[13,114],"of":[14,20,37,66,87],"dialogue":[15,31,88,94,108],"systems.":[16],"Evaluation,":[17],"in":[18,64],"itself,":[21],"is":[22],"a":[23,62],"crucial":[24],"part":[25],"during":[26],"development":[28],"process.":[29],"Often,":[30],"systems":[32,89,109],"are":[33],"evaluated":[34],"by":[35,100],"means":[36],"human":[38,67],"evaluations":[39],"questionnaires.":[41],"However,":[42],"tends":[44],"to":[45],"be":[46],"very":[47],"cost-":[48],"time-intensive.":[50],"Thus,":[51],"much":[52],"work":[53],"has":[54],"been":[55],"put":[56],"into":[57],"finding":[58],"which":[60],"allow":[61],"reduction":[63],"involvement":[65],"labour.":[68],"survey,":[71],"present":[73,112],"main":[75,103],"methods.":[78],"For":[79],"this,":[80],"differentiate":[82],"between":[83],"various":[85],"classes":[86],"(task-oriented,":[90],"conversational,":[91],"question-answering":[93],"systems).":[95],"We":[96],"cover":[97],"each":[98],"class":[99],"introducing":[101],"technologies":[104],"then":[111],"regarding":[116],"that":[117],"class.":[118]},"counts_by_year":[{"year":2026,"cited_by_count":23},{"year":2025,"cited_by_count":40},{"year":2024,"cited_by_count":35},{"year":2023,"cited_by_count":26},{"year":2022,"cited_by_count":36},{"year":2021,"cited_by_count":23},{"year":2020,"cited_by_count":4}],"updated_date":"2026-05-14T08:36:36.166977","created_date":"2019-05-16T00:00:00"}