{"id":"https://openalex.org/W4309618985","doi":"https://doi.org/10.1145/3555619","title":"Designing Conversational Evaluation Tools","display_name":"Designing Conversational Evaluation Tools","publication_year":2022,"publication_date":"2022-11-07","ids":{"openalex":"https://openalex.org/W4309618985","doi":"https://doi.org/10.1145/3555619"},"language":"en","primary_location":{"id":"doi:10.1145/3555619","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3555619","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://infoscience.epfl.ch/handle/20.500.14299/194263","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016878810","display_name":"Thiemo Wambsgan\u00df","orcid":"https://orcid.org/0000-0002-7440-9357"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":true,"raw_author_name":"Thiemo Wambsganss","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046196425","display_name":"Naim Zierau","orcid":"https://orcid.org/0000-0002-9451-8577"},"institutions":[{"id":"https://openalex.org/I202963720","display_name":"University of St.Gallen","ror":"https://ror.org/0561a3s31","country_code":"CH","type":"education","lineage":["https://openalex.org/I202963720"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Naim Zierau","raw_affiliation_strings":["University of St. Gallen, St. Gallen, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of St. Gallen, St. Gallen, Switzerland","institution_ids":["https://openalex.org/I202963720"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5045212721","display_name":"Matthias S\u00f6llner","orcid":"https://orcid.org/0000-0002-1347-8252"},"institutions":[{"id":"https://openalex.org/I106157433","display_name":"University of Kassel","ror":"https://ror.org/04zc7p361","country_code":"DE","type":"education","lineage":["https://openalex.org/I106157433"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Matthias S\u00f6llner","raw_affiliation_strings":["University of Kassel, Kassel, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Kassel, Kassel, Germany","institution_ids":["https://openalex.org/I106157433"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5007940211","display_name":"Tanja K\u00e4ser","orcid":"https://orcid.org/0000-0003-0672-0415"},"institutions":[{"id":"https://openalex.org/I5124864","display_name":"\u00c9cole Polytechnique F\u00e9d\u00e9rale de Lausanne","ror":"https://ror.org/02s376052","country_code":"CH","type":"education","lineage":["https://openalex.org/I2799323385","https://openalex.org/I5124864"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Tanja K\u00e4ser","raw_affiliation_strings":["EPFL, Lausanne, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"EPFL, Lausanne, Switzerland","institution_ids":["https://openalex.org/I5124864"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062550465","display_name":"Kenneth R. Koedinger","orcid":"https://orcid.org/0000-0002-5850-4768"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kenneth R. Koedinger","raw_affiliation_strings":["Carnegie Mellon University, Pittsburgh, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carnegie Mellon University, Pittsburgh, PA, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031171331","display_name":"Jan Marco Leimeister","orcid":"https://orcid.org/0000-0002-1990-2894"},"institutions":[{"id":"https://openalex.org/I202963720","display_name":"University of St.Gallen","ror":"https://ror.org/0561a3s31","country_code":"CH","type":"education","lineage":["https://openalex.org/I202963720"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Jan Marco Leimeister","raw_affiliation_strings":["University of St. Gallen & University of Kassel, St. Gallen, Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of St. Gallen & University of Kassel, St. Gallen, Switzerland","institution_ids":["https://openalex.org/I202963720"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5016878810"],"corresponding_institution_ids":["https://openalex.org/I5124864"],"apc_list":null,"apc_paid":null,"fwci":1.9421,"has_fulltext":false,"cited_by_count":15,"citation_normalized_percentile":{"value":0.88285189,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"6","issue":"CSCW2","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.98089998960495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6964147686958313},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.6427406072616577},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5809845924377441},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5739295482635498},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.5421574711799622},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.49148738384246826},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.4634997546672821},{"id":"https://openalex.org/keywords/dialog-system","display_name":"Dialog system","score":0.44705337285995483},{"id":"https://openalex.org/keywords/user-experience-design","display_name":"User experience design","score":0.4166494607925415},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3814949691295624},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.31721967458724976}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6964147686958313},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.6427406072616577},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5809845924377441},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5739295482635498},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.5421574711799622},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.49148738384246826},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.4634997546672821},{"id":"https://openalex.org/C190954187","wikidata":"https://www.wikidata.org/wiki/Q5270587","display_name":"Dialog system","level":3,"score":0.44705337285995483},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.4166494607925415},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3814949691295624},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.31721967458724976},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C173853756","wikidata":"https://www.wikidata.org/wiki/Q86915","display_name":"Dialog box","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3555619","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3555619","pdf_url":null,"source":{"id":"https://openalex.org/S4210183893","display_name":"Proceedings of the ACM on Human-Computer Interaction","issn_l":"2573-0142","issn":["2573-0142"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Human-Computer Interaction","raw_type":"journal-article"},{"id":"pmh:oai:infoscience.epfl.ch:20.500.14299/194263","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/194263","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"}],"best_oa_location":{"id":"pmh:oai:infoscience.epfl.ch:20.500.14299/194263","is_oa":true,"landing_page_url":"https://infoscience.epfl.ch/handle/20.500.14299/194263","pdf_url":null,"source":{"id":"https://openalex.org/S4306400487","display_name":"Infoscience (Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"conference proceedings"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8899999856948853}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":81,"referenced_works":["https://openalex.org/W198407522","https://openalex.org/W606329772","https://openalex.org/W1188413835","https://openalex.org/W1578907434","https://openalex.org/W1584405424","https://openalex.org/W1965322602","https://openalex.org/W1965686167","https://openalex.org/W1966797434","https://openalex.org/W1985824294","https://openalex.org/W1990520274","https://openalex.org/W1991015565","https://openalex.org/W1993092146","https://openalex.org/W1998259118","https://openalex.org/W1999297692","https://openalex.org/W2005388798","https://openalex.org/W2010398643","https://openalex.org/W2017036865","https://openalex.org/W2022619955","https://openalex.org/W2042003395","https://openalex.org/W2055666942","https://openalex.org/W2064001668","https://openalex.org/W2078658909","https://openalex.org/W2085230034","https://openalex.org/W2094244305","https://openalex.org/W2097516251","https://openalex.org/W2099995788","https://openalex.org/W2108752510","https://openalex.org/W2115040353","https://openalex.org/W2119681300","https://openalex.org/W2123984062","https://openalex.org/W2139006235","https://openalex.org/W2139087511","https://openalex.org/W2141158069","https://openalex.org/W2142822317","https://openalex.org/W2155069903","https://openalex.org/W2156897926","https://openalex.org/W2158821162","https://openalex.org/W2161307025","https://openalex.org/W2165542916","https://openalex.org/W2166073427","https://openalex.org/W2184015064","https://openalex.org/W2250338295","https://openalex.org/W2264742718","https://openalex.org/W2484145340","https://openalex.org/W2589387806","https://openalex.org/W2604452528","https://openalex.org/W2611049140","https://openalex.org/W2732780182","https://openalex.org/W2742631515","https://openalex.org/W2783929168","https://openalex.org/W2786250134","https://openalex.org/W2801795275","https://openalex.org/W2808264290","https://openalex.org/W2888925622","https://openalex.org/W2889547446","https://openalex.org/W2924456615","https://openalex.org/W2931579440","https://openalex.org/W2940530202","https://openalex.org/W2941035428","https://openalex.org/W2999981851","https://openalex.org/W3029044752","https://openalex.org/W3031500361","https://openalex.org/W3102980059","https://openalex.org/W3114396044","https://openalex.org/W3125868309","https://openalex.org/W3126302922","https://openalex.org/W3134155506","https://openalex.org/W3159367349","https://openalex.org/W3160042174","https://openalex.org/W3161418208","https://openalex.org/W3162076853","https://openalex.org/W3173895690","https://openalex.org/W4205184193","https://openalex.org/W4206379883","https://openalex.org/W4213388807","https://openalex.org/W4238846128","https://openalex.org/W4256088946","https://openalex.org/W4283398683","https://openalex.org/W4285245875","https://openalex.org/W4288080268","https://openalex.org/W4301407694"],"related_works":["https://openalex.org/W2385859805","https://openalex.org/W2530972254","https://openalex.org/W2374013449","https://openalex.org/W73545470","https://openalex.org/W2364381299","https://openalex.org/W2374430585","https://openalex.org/W2500779211","https://openalex.org/W48079147","https://openalex.org/W3144423903","https://openalex.org/W2250773064"],"abstract_inverted_index":{"Conversational":[0],"agents":[1],"(CAs)":[2],"provide":[3],"opportunities":[4],"for":[5,39],"improving":[6],"the":[7,65,73,81,94,102,105,121],"interaction":[8,90,107],"in":[9,72,93,136],"evaluation":[10,20,37,70],"surveys.":[11],"To":[12],"investigate":[13,80],"if":[14],"and":[15,26,86,133,141],"how":[16],"a":[17,33,43,53,84,129,137,146],"user-centered":[18],"conversational":[19,35,69,88,130],"tool":[21,38],"impacts":[22],"users'":[23],"response":[24,139],"quality":[25,112,140],"their":[27],"experience,":[28],"we":[29,49,78],"build":[30],"EVA":[31,51,92],"-":[32],"novel":[34],"course":[36],"educational":[40],"scenarios.":[41],"In":[42],"field":[44],"experiment":[45],"with":[46,109,114],"128":[47],"students,":[48],"compared":[50,119,144],"against":[52],"static":[54,147],"web":[55,148],"survey.":[56],"Our":[57,124],"results":[58,135],"confirm":[59],"prior":[60,100],"findings":[61,125],"from":[62],"literature":[63],"about":[64],"positive":[66],"effect":[67],"of":[68,75,91,104,117],"tools":[71],"domain":[74],"education.":[76],"Second,":[77],"then":[79],"differences":[82],"between":[83],"voice-based":[85,106],"text-based":[87,122],"human-computer":[89],"same":[95],"experimental":[96],"set-up.":[97],"Against":[98],"our":[99],"expectation,":[101],"students":[103],"answered":[108],"higher":[110,138],"information":[111,118],"but":[113],"lower":[115],"quantity":[116],"to":[120,145],"modality.":[123],"indicate":[126],"that":[127],"using":[128],"CA":[131],"(voice":[132],"text-based)":[134],"user":[142],"experience":[143],"survey":[149],"interface.":[150]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}