{"id":"https://openalex.org/W4408658796","doi":"https://doi.org/10.3390/informatics12010033","title":"Development of a Comprehensive Evaluation Scale for LLM-Powered Counseling Chatbots (CES-LCC) Using the eDelphi Method","display_name":"Development of a Comprehensive Evaluation Scale for LLM-Powered Counseling Chatbots (CES-LCC) Using the eDelphi Method","publication_year":2025,"publication_date":"2025-03-20","ids":{"openalex":"https://openalex.org/W4408658796","doi":"https://doi.org/10.3390/informatics12010033"},"language":"en","primary_location":{"id":"doi:10.3390/informatics12010033","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12010033","pdf_url":"https://www.mdpi.com/2227-9709/12/1/33/pdf?version=1742468602","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/2227-9709/12/1/33/pdf?version=1742468602","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033714227","display_name":"Marco Bolpagni","orcid":"https://orcid.org/0000-0001-7789-2535"},"institutions":[{"id":"https://openalex.org/I138689650","display_name":"University of Padua","ror":"https://ror.org/00240q980","country_code":"IT","type":"education","lineage":["https://openalex.org/I138689650"]},{"id":"https://openalex.org/I2277624104","display_name":"Fondazione Bruno Kessler","ror":"https://ror.org/01j33xk10","country_code":"IT","type":"facility","lineage":["https://openalex.org/I2277624104"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Marco Bolpagni","raw_affiliation_strings":["Department of General Psychology, University of Padova, 35121 Padova, Italy","Digital Health Research, Centre for Digital Health and Wellbeing, Fondazione Bruno Kessler, 38123 Trento, Italy"],"affiliations":[{"raw_affiliation_string":"Department of General Psychology, University of Padova, 35121 Padova, Italy","institution_ids":["https://openalex.org/I138689650"]},{"raw_affiliation_string":"Digital Health Research, Centre for Digital Health and Wellbeing, Fondazione Bruno Kessler, 38123 Trento, Italy","institution_ids":["https://openalex.org/I2277624104"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078229332","display_name":"Silvia Gabrielli","orcid":"https://orcid.org/0000-0002-7080-0615"},"institutions":[{"id":"https://openalex.org/I2277624104","display_name":"Fondazione Bruno Kessler","ror":"https://ror.org/01j33xk10","country_code":"IT","type":"facility","lineage":["https://openalex.org/I2277624104"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Silvia Gabrielli","raw_affiliation_strings":["Digital Health Research, Centre for Digital Health and Wellbeing, Fondazione Bruno Kessler, 38123 Trento, Italy"],"affiliations":[{"raw_affiliation_string":"Digital Health Research, Centre for Digital Health and Wellbeing, Fondazione Bruno Kessler, 38123 Trento, Italy","institution_ids":["https://openalex.org/I2277624104"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5033714227"],"corresponding_institution_ids":["https://openalex.org/I138689650","https://openalex.org/I2277624104"],"apc_list":{"value":1600,"currency":"CHF","value_usd":1732},"apc_paid":{"value":1600,"currency":"CHF","value_usd":1732},"fwci":20.0148,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.98810094,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"12","issue":"1","first_page":"33","last_page":"33"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10068","display_name":"Technology Adoption and User Behaviour","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9923999905586243,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11519","display_name":"Digital Mental Health Interventions","score":0.9818999767303467,"subfield":{"id":"https://openalex.org/subfields/3202","display_name":"Applied Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.6041097640991211},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.44445738196372986},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.389005571603775},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.3202447295188904},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.2868172526359558},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.125016450881958}],"concepts":[{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.6041097640991211},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.44445738196372986},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.389005571603775},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.3202447295188904},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2868172526359558},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.125016450881958},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.3390/informatics12010033","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12010033","pdf_url":"https://www.mdpi.com/2227-9709/12/1/33/pdf?version=1742468602","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:307f13c4d25646469ab7aa75c0519cfc","is_oa":true,"landing_page_url":"https://doaj.org/article/307f13c4d25646469ab7aa75c0519cfc","pdf_url":null,"source":{"id":"https://openalex.org/S112646816","display_name":"SHILAP Revista de lepidopterolog\u00eda","issn_l":"0300-5267","issn":["0300-5267","2340-4078"],"is_oa":true,"is_in_doaj":true,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Informatics, Vol 12, Iss 1, p 33 (2025)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/informatics12010033","is_oa":true,"landing_page_url":"https://doi.org/10.3390/informatics12010033","pdf_url":"https://www.mdpi.com/2227-9709/12/1/33/pdf?version=1742468602","source":{"id":"https://openalex.org/S2738238905","display_name":"Informatics","issn_l":"2227-9709","issn":["2227-9709"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Informatics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408658796.pdf"},"referenced_works_count":70,"referenced_works":["https://openalex.org/W1777152392","https://openalex.org/W1791587663","https://openalex.org/W1970200091","https://openalex.org/W1977052699","https://openalex.org/W2035944149","https://openalex.org/W2061935394","https://openalex.org/W2062299969","https://openalex.org/W2100379340","https://openalex.org/W2101105183","https://openalex.org/W2159306398","https://openalex.org/W2260766877","https://openalex.org/W2342249984","https://openalex.org/W2481365383","https://openalex.org/W2587777701","https://openalex.org/W2765110977","https://openalex.org/W2886678476","https://openalex.org/W2920269499","https://openalex.org/W2947017203","https://openalex.org/W2964311550","https://openalex.org/W2971921133","https://openalex.org/W3007384022","https://openalex.org/W3012939091","https://openalex.org/W3119676664","https://openalex.org/W3121308480","https://openalex.org/W3138299337","https://openalex.org/W3159362230","https://openalex.org/W3173498569","https://openalex.org/W3187911027","https://openalex.org/W3208401146","https://openalex.org/W4200110762","https://openalex.org/W4200615999","https://openalex.org/W4233023712","https://openalex.org/W4234580964","https://openalex.org/W4246277771","https://openalex.org/W4250223980","https://openalex.org/W4255441728","https://openalex.org/W4281489185","https://openalex.org/W4290047590","https://openalex.org/W4292230838","https://openalex.org/W4300861424","https://openalex.org/W4311675890","https://openalex.org/W4312118598","https://openalex.org/W4313266830","https://openalex.org/W4318818619","https://openalex.org/W4367047335","https://openalex.org/W4384520759","https://openalex.org/W4384523644","https://openalex.org/W4385135972","https://openalex.org/W4386330335","https://openalex.org/W4386566618","https://openalex.org/W4387494581","https://openalex.org/W4387970754","https://openalex.org/W4389504244","https://openalex.org/W4391136507","https://openalex.org/W4391932545","https://openalex.org/W4392011662","https://openalex.org/W4394746513","https://openalex.org/W4394967854","https://openalex.org/W4396833024","https://openalex.org/W4400089271","https://openalex.org/W4400230936","https://openalex.org/W4401082653","https://openalex.org/W4402909345","https://openalex.org/W4403247877","https://openalex.org/W4405833216","https://openalex.org/W6638161682","https://openalex.org/W6682631176","https://openalex.org/W6692485200","https://openalex.org/W6760132758","https://openalex.org/W6788910947"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Background/Objectives:":[0],"With":[1],"advancements":[2],"in":[3,79,89,212],"Large":[4],"Language":[5,167],"Models":[6],"(LLMs),":[7],"counseling":[8,217],"chatbots":[9],"are":[10,193,231],"becoming":[11],"essential":[12],"tools":[13],"for":[14,60,140,195,223],"delivering":[15,130],"scalable":[16],"and":[17,42,52,85,100,105,136,163,173,176,201,244],"accessible":[18],"mental":[19],"health":[20],"support.":[21],"Traditional":[22],"evaluation":[23,54,214],"scales,":[24],"however,":[25],"fail":[26],"to":[27,48,69,125,237],"adequately":[28],"capture":[29],"the":[30,56,66,113,117,123,148,213,239],"sophisticated":[31],"capabilities":[32],"of":[33,76,116,147,165,203,215],"these":[34],"systems,":[35],"such":[36,198],"as":[37,199],"personalized":[38],"interactions,":[39],"empathetic":[40],"responses,":[41],"memory":[43],"retention.":[44],"This":[45],"study":[46],"aims":[47],"design":[49],"a":[50,209,220],"robust":[51],"comprehensive":[53],"scale,":[55,118],"Comprehensive":[57],"Evaluation":[58],"Scale":[59],"LLM-Powered":[61],"Counseling":[62],"Chatbots":[63],"(CES-LCC),":[64],"using":[65,122],"eDelphi":[67,92],"method":[68],"address":[70],"this":[71],"gap.":[72],"Methods:":[73],"A":[74],"panel":[75],"16":[77],"experts":[78],"psychology,":[80],"artificial":[81],"intelligence,":[82],"human-computer":[83],"interaction,":[84],"digital":[86],"therapeutics":[87],"participated":[88],"two":[90],"iterative":[91],"rounds.":[93],"The":[94,144,206],"process":[95],"focused":[96],"on":[97,103],"refining":[98],"dimensions":[99],"items":[101,152],"based":[102],"qualitative":[104],"quantitative":[106],"feedback.":[107],"Initial":[108,179],"validation,":[109],"conducted":[110],"after":[111],"assembling":[112],"final":[114,145],"version":[115,146],"involved":[119],"49":[120],"participants":[121],"CES-LCC":[124,149,207],"evaluate":[126],"an":[127,134],"LLM-powered":[128,216],"chatbot":[129],"Self-Help":[131],"Plus":[132],"(SH+),":[133],"Acceptance":[135],"Commitment":[137],"Therapy-based":[138],"intervention":[139],"stress":[141],"management.":[142],"Results:":[143],"features":[150],"27":[151],"grouped":[153],"into":[154],"nine":[155],"dimensions:":[156],"Understanding":[157],"Requests,":[158],"Providing":[159],"Helpful":[160],"Information,":[161],"Clarity":[162,200],"Relevance":[164,202],"Responses,":[166],"Quality,":[168],"Trust,":[169],"Emotional":[170],"Support,":[171],"Guidance":[172],"Direction,":[174],"Memory,":[175],"Overall":[177],"Satisfaction.":[178],"real-world":[180],"validation":[181],"revealed":[182],"high":[183],"internal":[184],"consistency":[185],"(Cronbach\u2019s":[186],"alpha":[187],"=":[188],"0.94),":[189],"although":[190],"minor":[191],"adjustments":[192],"required":[194],"specific":[196],"dimensions,":[197],"Responses.":[204],"Conclusions:":[205],"fills":[208],"critical":[210],"gap":[211],"chatbots,":[218],"offering":[219],"standardized":[221],"tool":[222],"assessing":[224],"their":[225],"multifaceted":[226],"capabilities.":[227],"While":[228],"preliminary":[229],"results":[230],"promising,":[232],"further":[233],"research":[234],"is":[235],"needed":[236],"validate":[238],"scale":[240],"across":[241],"diverse":[242],"populations":[243],"settings.":[245]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3}],"updated_date":"2026-04-07T14:57:38.498316","created_date":"2025-10-10T00:00:00"}