{"id":"https://openalex.org/W4206083821","doi":"https://doi.org/10.1109/access.2022.3145323","title":"Experimentation for Chatbot Usability Evaluation: A Secondary Study","display_name":"Experimentation for Chatbot Usability Evaluation: A Secondary Study","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4206083821","doi":"https://doi.org/10.1109/access.2022.3145323"},"language":"en","primary_location":{"id":"doi:10.1109/access.2022.3145323","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3145323","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09687577.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09687577.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5008829475","display_name":"Ranci Ren","orcid":null},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Ranci Ren","raw_affiliation_strings":["Escuela Polit\u00e9cnica Superior, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Escuela Polit\u00e9cnica Superior, Universidad Aut\u00f3noma de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006090595","display_name":"Mireya Zapata","orcid":"https://orcid.org/0000-0003-3382-2724"},"institutions":[{"id":"https://openalex.org/I4210127378","display_name":"Universidad Indoam\u00e9rica","ror":"https://ror.org/02nn67a65","country_code":"EC","type":"education","lineage":["https://openalex.org/I4210127378"]}],"countries":["EC"],"is_corresponding":false,"raw_author_name":"Mireya Zapata","raw_affiliation_strings":["Research Center of Mechatronics and Interactive Systems (MIST), Universidad Tecnol\u00f3gica Indoam\u00e9rica, Quito, Ecuador"],"raw_orcid":"https://orcid.org/0000-0003-3382-2724","affiliations":[{"raw_affiliation_string":"Research Center of Mechatronics and Interactive Systems (MIST), Universidad Tecnol\u00f3gica Indoam\u00e9rica, Quito, Ecuador","institution_ids":["https://openalex.org/I4210127378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090326213","display_name":"John W. Castro","orcid":"https://orcid.org/0000-0002-7938-7564"},"institutions":[{"id":"https://openalex.org/I201487725","display_name":"University of Atacama","ror":"https://ror.org/022yres73","country_code":"CL","type":"education","lineage":["https://openalex.org/I201487725"]}],"countries":["CL"],"is_corresponding":false,"raw_author_name":"John W. Castro","raw_affiliation_strings":["Universidad de Atacama, Copiap\u00f3, Chile"],"raw_orcid":"https://orcid.org/0000-0002-7938-7564","affiliations":[{"raw_affiliation_string":"Universidad de Atacama, Copiap\u00f3, Chile","institution_ids":["https://openalex.org/I201487725"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004737603","display_name":"\u00d3scar Dieste","orcid":"https://orcid.org/0000-0002-3060-7853"},"institutions":[{"id":"https://openalex.org/I88060688","display_name":"Universidad Polit\u00e9cnica de Madrid","ror":"https://ror.org/03n6nwv02","country_code":"ES","type":"education","lineage":["https://openalex.org/I88060688"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Oscar Dieste","raw_affiliation_strings":["Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain"],"raw_orcid":"https://orcid.org/0000-0002-3060-7853","affiliations":[{"raw_affiliation_string":"Escuela T\u00e9cnica Superior de Ingenieros Inform\u00e1ticos, Universidad Polit\u00e9cnica de Madrid, Boadilla del Monte, Spain","institution_ids":["https://openalex.org/I88060688"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022941249","display_name":"Silvia T. Acu\u00f1a","orcid":"https://orcid.org/0000-0003-2135-6113"},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Silvia T. Acuna","raw_affiliation_strings":["Escuela Polit\u00e9cnica Superior, Universidad Aut\u00f3noma de Madrid, Madrid, Spain"],"raw_orcid":"https://orcid.org/0000-0003-2135-6113","affiliations":[{"raw_affiliation_string":"Escuela Polit\u00e9cnica Superior, Universidad Aut\u00f3noma de Madrid, Madrid, Spain","institution_ids":["https://openalex.org/I63634437"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":4.9876,"has_fulltext":true,"cited_by_count":49,"citation_normalized_percentile":{"value":0.95797948,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"10","issue":null,"first_page":"12430","last_page":"12464"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11675","display_name":"Open Source Software Innovations","score":0.9824000000953674,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14246","display_name":"E-Learning and Knowledge Management","score":0.9818999767303467,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/chatbot","display_name":"Chatbot","score":0.9724322557449341},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.9293999671936035},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.749604344367981},{"id":"https://openalex.org/keywords/usability-lab","display_name":"Usability lab","score":0.5102033615112305},{"id":"https://openalex.org/keywords/usability-inspection","display_name":"Usability inspection","score":0.5086868405342102},{"id":"https://openalex.org/keywords/usability-engineering","display_name":"Usability engineering","score":0.504254937171936},{"id":"https://openalex.org/keywords/web-usability","display_name":"Web usability","score":0.48739248514175415},{"id":"https://openalex.org/keywords/usability-goals","display_name":"Usability goals","score":0.4738624095916748},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.42789506912231445},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.4027189612388611}],"concepts":[{"id":"https://openalex.org/C2779041454","wikidata":"https://www.wikidata.org/wiki/Q870780","display_name":"Chatbot","level":2,"score":0.9724322557449341},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.9293999671936035},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.749604344367981},{"id":"https://openalex.org/C11324603","wikidata":"https://www.wikidata.org/wiki/Q2502322","display_name":"Usability lab","level":4,"score":0.5102033615112305},{"id":"https://openalex.org/C23456302","wikidata":"https://www.wikidata.org/wiki/Q7901668","display_name":"Usability inspection","level":4,"score":0.5086868405342102},{"id":"https://openalex.org/C100302975","wikidata":"https://www.wikidata.org/wiki/Q1642623","display_name":"Usability engineering","level":3,"score":0.504254937171936},{"id":"https://openalex.org/C4237393","wikidata":"https://www.wikidata.org/wiki/Q1636686","display_name":"Web usability","level":3,"score":0.48739248514175415},{"id":"https://openalex.org/C62993174","wikidata":"https://www.wikidata.org/wiki/Q2928808","display_name":"Usability goals","level":4,"score":0.4738624095916748},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.42789506912231445},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.4027189612388611}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2022.3145323","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3145323","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09687577.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:92091a9453894eb38133fdc8373d1d0c","is_oa":true,"landing_page_url":"https://doaj.org/article/92091a9453894eb38133fdc8373d1d0c","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 10, Pp 12430-12464 (2022)","raw_type":"article"},{"id":"pmh:oai:repositorio.uam.es:10486/711122","is_oa":true,"landing_page_url":"http://hdl.handle.net/10486/711122","pdf_url":null,"source":{"id":"https://openalex.org/S4306400964","display_name":"Biblos-e Archivo (Universidad Aut\u00f3noma de Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I63634437","host_organization_name":"Universidad Aut\u00f3noma de Madrid","host_organization_lineage":["https://openalex.org/I63634437"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"doi:10.1109/access.2022.3145323","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2022.3145323","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/9668973/09687577.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","score":0.6399999856948853,"id":"https://metadata.un.org/sdg/4"}],"awards":[{"id":"https://openalex.org/G3458438125","display_name":null,"funder_award_id":"P2018/TCS-4314","funder_id":"https://openalex.org/F4320324004","funder_display_name":"Forskningsr\u00e5det om H\u00e4lsa, Arbetsliv och V\u00e4lf\u00e4rd"},{"id":"https://openalex.org/G7783118353","display_name":null,"funder_award_id":"PGC2018-097265-B-I00","funder_id":"https://openalex.org/F4320315062","funder_display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades"},{"id":"https://openalex.org/G8063416118","display_name":null,"funder_award_id":"PGC2018","funder_id":"https://openalex.org/F4320315062","funder_display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades"},{"id":"https://openalex.org/G8181105605","display_name":null,"funder_award_id":"RTI2018","funder_id":"https://openalex.org/F4320315062","funder_display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades"}],"funders":[{"id":"https://openalex.org/F4320315062","display_name":"Ministerio de Ciencia, Innovaci\u00f3n y Universidades","ror":null},{"id":"https://openalex.org/F4320324004","display_name":"Forskningsr\u00e5det om H\u00e4lsa, Arbetsliv och V\u00e4lf\u00e4rd","ror":"https://ror.org/02d290r06"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4206083821.pdf","grobid_xml":"https://content.openalex.org/works/W4206083821.grobid-xml"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W4214443","https://openalex.org/W613277099","https://openalex.org/W1841651683","https://openalex.org/W1876404477","https://openalex.org/W2003323761","https://openalex.org/W2050118439","https://openalex.org/W2053154970","https://openalex.org/W2101623580","https://openalex.org/W2122909041","https://openalex.org/W2133940850","https://openalex.org/W2164777277","https://openalex.org/W2167926541","https://openalex.org/W2222168624","https://openalex.org/W2516963373","https://openalex.org/W2589415147","https://openalex.org/W2612764117","https://openalex.org/W2795758037","https://openalex.org/W2889730315","https://openalex.org/W2908463346","https://openalex.org/W2912404447","https://openalex.org/W2946465872","https://openalex.org/W2963392119","https://openalex.org/W2990003990","https://openalex.org/W2996421693","https://openalex.org/W2999746425","https://openalex.org/W3005639702","https://openalex.org/W3006230258","https://openalex.org/W3008487900","https://openalex.org/W3021626742","https://openalex.org/W3021951675","https://openalex.org/W3023605951","https://openalex.org/W3082474355","https://openalex.org/W3103891289","https://openalex.org/W3135939776","https://openalex.org/W3139433464","https://openalex.org/W3160782978","https://openalex.org/W4234669218","https://openalex.org/W4239350751","https://openalex.org/W4253001350","https://openalex.org/W4288073892","https://openalex.org/W4291213652","https://openalex.org/W4297813086","https://openalex.org/W6736461433","https://openalex.org/W6754659666","https://openalex.org/W6758498913","https://openalex.org/W6770638308"],"related_works":["https://openalex.org/W110518437","https://openalex.org/W2488259327","https://openalex.org/W2789812950","https://openalex.org/W4230862284","https://openalex.org/W2560659645","https://openalex.org/W4249900088","https://openalex.org/W2100424850","https://openalex.org/W2075732570","https://openalex.org/W2580784735","https://openalex.org/W2120843845"],"abstract_inverted_index":{"Interest":[0],"in":[1,16,71,83,108,205,221],"chatbot":[2,17,25,84,215],"development":[3],"is":[4,12,40,74,199,224],"on":[5,24],"the":[6,19,55,60,67,78,81,95,104,117,129,167,170,202,246,251],"rise.":[7],"As":[8,31],"a":[9,32,36,89,113,135,182],"usability":[10,26,68,85,105,144,216,238],"evaluation":[11],"an":[13,212],"essential":[14],"step":[15],"development,":[18],"number":[20],"of":[21,69,80,106,128,134,137,189,207,214,250],"experimental":[22,171,252],"studies":[23],"has":[27],"grown":[28],"as":[29,201,227],"well.":[30],"result,":[33],"we":[34],"think":[35],"systematic":[37,90],"mapping":[38,91],"study":[39,92],"opportune.":[41],"We":[42,87],"analyzed":[43],"more":[44,242],"than":[45],"700":[46],"sources":[47],"and":[48,58,62,99,125,141,154,161,185,191,248],"retrieved":[49],"28":[50],"primary":[51],"studies.":[52],"By":[53],"aggregating":[54],"research":[56,96,210],"questions":[57],"examining":[59],"characteristics":[61,145],"metrics":[63,100],"used":[64,101,146],"to":[65,76,93,102,147,157,232,244],"evaluate":[66,103],"chatbots":[70,107,156,175],"experiments,":[72],"it":[73],"possible":[75],"identify":[77,94,148],"state":[79],"art":[82],"experimentation.":[86,217],"conducted":[88],"questions,":[97],"characteristics,":[98],"experiments.":[109,138],"Most":[110],"experiments":[111,121,239],"adopted":[112],"within-subjects":[114],"design.":[115],"On":[116],"other":[118],"hand,":[119],"few":[120],"provided":[122],"raw":[123],"data,":[124],"only":[126],"one":[127],"identified":[130],"papers":[131],"was":[132],"part":[133],"family":[136],"Effectiveness,":[139],"efficiency,":[140],"satisfaction":[142],"are":[143,165],"how":[149,162],"well":[150],"users":[151,164],"can":[152],"learn":[153],"use":[155],"achieve":[158],"their":[159],"goals":[160],"satisfied":[163],"during":[166],"interaction.":[168],"Generally,":[169],"results":[172],"revealed":[173],"that":[174],"have":[176],"several":[177],"advantages":[178],"(e.g.,":[179,194],"they":[180,186],"provide":[181],"real-time":[183],"response":[184],"improve":[187,245],"ease":[188],"use)":[190],"some":[192],"shortcomings":[193],"natural":[195],"language":[196],"processing,":[197],"which":[198],"rated":[200],"weakness":[203],"most":[204],"need":[206],"improvement).":[208],"This":[209],"offers":[211],"overview":[213],"The":[218],"increasing":[219],"interest":[220],"this":[222],"area":[223],"very":[225],"recent,":[226],"works":[228],"did":[229],"not":[230],"start":[231],"be":[233,241],"published":[234],"until":[235],"2018.":[236],"Chatbot":[237],"should":[240],"replicable":[243],"reliability":[247],"transparency":[249],"results.":[253]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":10},{"year":2022,"cited_by_count":2}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}