{"id":"https://openalex.org/W3205880454","doi":"https://doi.org/10.3390/info12110437","title":"Quality Assessment Methods for Textual Conversational Interfaces: A Multivocal Literature Review","display_name":"Quality Assessment Methods for Textual Conversational Interfaces: A Multivocal Literature Review","publication_year":2021,"publication_date":"2021-10-21","ids":{"openalex":"https://openalex.org/W3205880454","doi":"https://doi.org/10.3390/info12110437","mag":"3205880454"},"language":"en","primary_location":{"id":"doi:10.3390/info12110437","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info12110437","pdf_url":"https://www.mdpi.com/2078-2489/12/11/437/pdf?version=1635838095","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2078-2489/12/11/437/pdf?version=1635838095","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040640308","display_name":"Riccardo Coppola","orcid":"https://orcid.org/0000-0003-4601-7425"},"institutions":[{"id":"https://openalex.org/I177477856","display_name":"Politecnico di Torino","ror":"https://ror.org/00bgk9508","country_code":"IT","type":"education","lineage":["https://openalex.org/I177477856"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Riccardo Coppola","raw_affiliation_strings":["Dipartimento di Automatica ed Informatica (DAUIN), Politecnico di Torino, Corso Duca degli Abruzzi 24, 10129 Torino, Italy"],"affiliations":[{"raw_affiliation_string":"Dipartimento di Automatica ed Informatica (DAUIN), Politecnico di Torino, Corso Duca degli Abruzzi 24, 10129 Torino, Italy","institution_ids":["https://openalex.org/I177477856"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040165772","display_name":"Luca Ardito","orcid":"https://orcid.org/0000-0002-0501-7886"},"institutions":[{"id":"https://openalex.org/I177477856","display_name":"Politecnico di Torino","ror":"https://ror.org/00bgk9508","country_code":"IT","type":"education","lineage":["https://openalex.org/I177477856"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Luca Ardito","raw_affiliation_strings":["Dipartimento di Automatica ed Informatica (DAUIN), Politecnico di Torino, Corso Duca degli Abruzzi 24, 10129 Torino, Italy"],"affiliations":[{"raw_affiliation_string":"Dipartimento di Automatica ed Informatica (DAUIN), Politecnico di Torino, Corso Duca degli Abruzzi 24, 10129 Torino, Italy","institution_ids":["https://openalex.org/I177477856"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5040165772"],"corresponding_institution_ids":["https://openalex.org/I177477856"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.5599,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.7392177,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"12","issue":"11","first_page":"437","last_page":"437"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.991599977016449,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7518728971481323},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.7302682399749756},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5716930627822876},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5669195055961609},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.47155189514160156},{"id":"https://openalex.org/keywords/systematic-review","display_name":"Systematic review","score":0.4669037461280823},{"id":"https://openalex.org/keywords/macro","display_name":"Macro","score":0.4385090470314026},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3883932828903198},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34773391485214233},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3368508815765381},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.29676055908203125}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7518728971481323},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.7302682399749756},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5716930627822876},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5669195055961609},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.47155189514160156},{"id":"https://openalex.org/C189708586","wikidata":"https://www.wikidata.org/wiki/Q1504425","display_name":"Systematic review","level":3,"score":0.4669037461280823},{"id":"https://openalex.org/C166955791","wikidata":"https://www.wikidata.org/wiki/Q629579","display_name":"Macro","level":2,"score":0.4385090470314026},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3883932828903198},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34773391485214233},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3368508815765381},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29676055908203125},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C2779473830","wikidata":"https://www.wikidata.org/wiki/Q1540899","display_name":"MEDLINE","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.3390/info12110437","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info12110437","pdf_url":"https://www.mdpi.com/2078-2489/12/11/437/pdf?version=1635838095","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8ce15c5efd6241b6bf23093b83bb1edf","is_oa":true,"landing_page_url":"https://doaj.org/article/8ce15c5efd6241b6bf23093b83bb1edf","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information, Vol 12, Iss 11, p 437 (2021)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/2078-2489/12/11/437/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/info12110437","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Information; Volume 12; Issue 11; Pages: 437","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/info12110437","is_oa":true,"landing_page_url":"https://doi.org/10.3390/info12110437","pdf_url":"https://www.mdpi.com/2078-2489/12/11/437/pdf?version=1635838095","source":{"id":"https://openalex.org/S4210219776","display_name":"Information","issn_l":"2078-2489","issn":["2078-2489"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Information","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.8399999737739563,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G2066797808","display_name":null,"funder_award_id":"952652","funder_id":"https://openalex.org/F4320320300","funder_display_name":"European Commission"},{"id":"https://openalex.org/G4320622773","display_name":null,"funder_award_id":"952652","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"}],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3205880454.pdf","grobid_xml":"https://content.openalex.org/works/W3205880454.grobid-xml"},"referenced_works_count":121,"referenced_works":["https://openalex.org/W4214443","https://openalex.org/W198151952","https://openalex.org/W1511279896","https://openalex.org/W1531131521","https://openalex.org/W1537988608","https://openalex.org/W1607675442","https://openalex.org/W1659431435","https://openalex.org/W1774811845","https://openalex.org/W1973592082","https://openalex.org/W1985856979","https://openalex.org/W1991658485","https://openalex.org/W1999024121","https://openalex.org/W2008155708","https://openalex.org/W2019955851","https://openalex.org/W2033317224","https://openalex.org/W2056698510","https://openalex.org/W2069074793","https://openalex.org/W2082372845","https://openalex.org/W2101105183","https://openalex.org/W2108025693","https://openalex.org/W2108835136","https://openalex.org/W2123952992","https://openalex.org/W2138571267","https://openalex.org/W2139196777","https://openalex.org/W2139786076","https://openalex.org/W2151162056","https://openalex.org/W2181505431","https://openalex.org/W2185604715","https://openalex.org/W2202602917","https://openalex.org/W2250640299","https://openalex.org/W2328332630","https://openalex.org/W2336383631","https://openalex.org/W2345039647","https://openalex.org/W2405640850","https://openalex.org/W2407772516","https://openalex.org/W2419539795","https://openalex.org/W2484054226","https://openalex.org/W2519974261","https://openalex.org/W2567362164","https://openalex.org/W2576494997","https://openalex.org/W2589387806","https://openalex.org/W2612764117","https://openalex.org/W2622863697","https://openalex.org/W2635153012","https://openalex.org/W2735864825","https://openalex.org/W2750868684","https://openalex.org/W2753691335","https://openalex.org/W2761423999","https://openalex.org/W2787579701","https://openalex.org/W2790351125","https://openalex.org/W2790609525","https://openalex.org/W2795758037","https://openalex.org/W2798385473","https://openalex.org/W2798888952","https://openalex.org/W2808216081","https://openalex.org/W2888929421","https://openalex.org/W2891717274","https://openalex.org/W2892322896","https://openalex.org/W2900107703","https://openalex.org/W2900726863","https://openalex.org/W2908285010","https://openalex.org/W2916904544","https://openalex.org/W2945210383","https://openalex.org/W2946465872","https://openalex.org/W2947017203","https://openalex.org/W2951035168","https://openalex.org/W2954209768","https://openalex.org/W2954478324","https://openalex.org/W2955341878","https://openalex.org/W2956203845","https://openalex.org/W2962532739","https://openalex.org/W2962854379","https://openalex.org/W2962974452","https://openalex.org/W2963332597","https://openalex.org/W2963527228","https://openalex.org/W2963903950","https://openalex.org/W2964543620","https://openalex.org/W2965761667","https://openalex.org/W2966245214","https://openalex.org/W2967393852","https://openalex.org/W2978657664","https://openalex.org/W2999333374","https://openalex.org/W3003696541","https://openalex.org/W3006086281","https://openalex.org/W3019577029","https://openalex.org/W3022629558","https://openalex.org/W3044483028","https://openalex.org/W3047240015","https://openalex.org/W3082622239","https://openalex.org/W3084232758","https://openalex.org/W3090127949","https://openalex.org/W3090684352","https://openalex.org/W3100468535","https://openalex.org/W3113561521","https://openalex.org/W3119946054","https://openalex.org/W3122379246","https://openalex.org/W3127356397","https://openalex.org/W3127685069","https://openalex.org/W3128074023","https://openalex.org/W3133035511","https://openalex.org/W3137075421","https://openalex.org/W3138864873","https://openalex.org/W3153726076","https://openalex.org/W3157200701","https://openalex.org/W3160418499","https://openalex.org/W3161307653","https://openalex.org/W3174382772","https://openalex.org/W3176254669","https://openalex.org/W3184229183","https://openalex.org/W3197876086","https://openalex.org/W3198454285","https://openalex.org/W4236521339","https://openalex.org/W4249482913","https://openalex.org/W4292361572","https://openalex.org/W6667939506","https://openalex.org/W6677154328","https://openalex.org/W6684439845","https://openalex.org/W6701963771","https://openalex.org/W6807623708","https://openalex.org/W6812932845","https://openalex.org/W7045675062"],"related_works":["https://openalex.org/W2165912799","https://openalex.org/W2735662278","https://openalex.org/W2382615723","https://openalex.org/W4405331580","https://openalex.org/W4311804456","https://openalex.org/W1987484445","https://openalex.org/W2623658258","https://openalex.org/W2030816003","https://openalex.org/W2143413548","https://openalex.org/W1969219540"],"abstract_inverted_index":{"The":[0],"evaluation":[1,43,46,155],"and":[2,45,55,73,83,89,95,100,134,170,196,203],"assessment":[3],"of":[4,57,67,115,132,146,156,161,168,182],"conversational":[5,157,183],"interfaces":[6],"is":[7],"a":[8,25,38,64,113,165,190],"complex":[9],"task":[10],"since":[11],"such":[12],"software":[13],"products":[14],"are":[15,103,173],"challenging":[16],"to":[17,36,49,52,128,178,198,206],"validate":[18],"through":[19],"traditional":[20],"testing":[21],"approaches.":[22],"We":[23,60,77,121],"conducted":[24],"systematic":[26],"Multivocal":[27],"Literature":[28],"Review":[29],"(MLR),":[30],"on":[31,40,143],"five":[32,124],"different":[33,80,87,125,140],"literature":[34,75,162,177],"sources,":[35],"provide":[37,50],"view":[39],"quality":[41,81],"attributes,":[42],"frameworks,":[44],"datasets":[47,136],"proposed":[48],"aid":[51],"the":[53,58,108,144,154,176,180,200],"researchers":[54,195],"practitioners":[56,197],"field.":[59],"came":[61],"up":[62],"with":[63],"final":[65],"pool":[66],"118":[68],"contributions,":[69],"including":[70],"grey":[71],"(35)":[72],"white":[74],"(83).":[76],"categorized":[78],"123":[79],"attributes":[82,102,172],"metrics":[84],"under":[85],"ten":[86],"categories":[88,141],"four":[90],"macro-categories:":[91],"Relational,":[92],"Conversational,":[93],"User-Centered":[94,116],"Quantitative":[96],"attributes.":[97],"While":[98],"Relational":[99],"Conversational":[101],"most":[104],"commonly":[105],"explored":[106],"by":[107],"scientific":[109],"literature,":[110],"we":[111],"testified":[112],"predominance":[114],"Attributes":[117],"in":[118,175],"industrial":[119],"literature.":[120],"also":[122],"identified":[123],"academic":[126],"frameworks/tools":[127],"automatically":[129],"compute":[130],"sets":[131],"metrics,":[133],"28":[135],"(subdivided":[137],"into":[138],"seven":[139],"based":[142],"type":[145],"data":[147],"contained)":[148],"that":[149,164],"can":[150,187],"produce":[151],"conversations":[152],"for":[153,194,209],"interfaces.":[158,184],"Our":[159,185],"analysis":[160],"highlights":[163],"high":[166],"number":[167],"qualitative":[169],"quantitative":[171],"available":[174],"evaluate":[179],"performance":[181],"categorization":[186],"serve":[188],"as":[189],"valid":[191],"entry":[192],"point":[193],"select":[199],"proper":[201],"functional":[202],"non-functional":[204],"aspects":[205],"be":[207],"evaluated":[208],"their":[210],"products.":[211]},"counts_by_year":[{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2021-10-25T00:00:00"}