{"id":"https://openalex.org/W7166804539","doi":"https://doi.org/10.18653/v1/2026.clpsych-1.24","title":"Mostly Grounded, Occasionally Risky: Expert Evaluation of LLM-Generated Supervisory Feedback in a Psychotherapy Training Simulator","display_name":"Mostly Grounded, Occasionally Risky: Expert Evaluation of LLM-Generated Supervisory Feedback in a Psychotherapy Training Simulator","publication_year":2026,"publication_date":"2026-01-01","ids":{"openalex":"https://openalex.org/W7166804539","doi":"https://doi.org/10.18653/v1/2026.clpsych-1.24"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2026.clpsych-1.24","doi":"https://doi.org/10.18653/v1/2026.clpsych-1.24","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.clpsych-1.24","pdf_url":"https://aclanthology.org/2026.clpsych-1.24.pdf","source":{"id":"https://openalex.org/S7407087913","display_name":"Workshop on Computational Linguistics and Clinical Psychology (CLPsych)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th Workshop on Computational Linguistics and Clinical Psychology (CLPsych 2026)","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2026.clpsych-1.24.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046420206","display_name":"Adri\u00e1n Montesano","orcid":"https://orcid.org/0000-0001-9201-8694","observed_orcids":["https://orcid.org/0000-0001-9201-8694"]},"institutions":[{"id":"https://openalex.org/I138847295","display_name":"Universitat Oberta de Catalunya","ror":"https://ror.org/01f5wp925","country_code":"ES","type":"education","lineage":["https://openalex.org/I138847295"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Adrian Montesano","raw_affiliation_strings":["Open University of Catalonia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open University of Catalonia","institution_ids":["https://openalex.org/I138847295"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5119593922","display_name":"Justin K. Bloomberg","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I138847295","display_name":"Universitat Oberta de Catalunya","ror":"https://ror.org/01f5wp925","country_code":"ES","type":"education","lineage":["https://openalex.org/I138847295"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Justin Bloomberg","raw_affiliation_strings":["Open University of Catalonia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open University of Catalonia","institution_ids":["https://openalex.org/I138847295"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5037597009","display_name":"Marc P\u00e9rez Burriel","orcid":"https://orcid.org/0000-0001-9147-5498","observed_orcids":["https://orcid.org/0000-0001-9147-5498"]},"institutions":[{"id":"https://openalex.org/I251424209","display_name":"Universitat de Girona","ror":"https://ror.org/01xdxns91","country_code":"ES","type":"education","lineage":["https://openalex.org/I251424209"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Marc P\u00e9rez-Buriel","raw_affiliation_strings":["Universitat de Girona"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Universitat de Girona","institution_ids":["https://openalex.org/I251424209"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.44395924,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"298","last_page":"305"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8744999766349792,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.8744999766349792,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.04179999977350235,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10214","display_name":"Psychotherapy Techniques and Applications","score":0.010400000028312206,"subfield":{"id":"https://openalex.org/subfields/3203","display_name":"Clinical Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/llm-generated-feedback","display_name":"LLM-generated feedback","score":0.8569999933242798},{"id":"https://openalex.org/keywords/supervisor-feedback","display_name":"supervisor feedback","score":0.8149999976158142},{"id":"https://openalex.org/keywords/psychotherapy-training","display_name":"psychotherapy training","score":0.8529999852180481},{"id":"https://openalex.org/keywords/training-simulators","display_name":"training simulators","score":0.6389999985694885},{"id":"https://openalex.org/keywords/expert-evaluation","display_name":"expert evaluation","score":0.7620000243186951},{"id":"https://openalex.org/keywords/novice-trainees","display_name":"novice trainees","score":0.6349999904632568},{"id":"https://openalex.org/keywords/clinical-inference","display_name":"clinical inference","score":0.4580000042915344},{"id":"https://openalex.org/keywords/pragmatic-failure","display_name":"pragmatic failure","score":0.7689999938011169}],"concepts":[{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.5593000054359436},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5555999875068665},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.35339999198913574},{"id":"https://openalex.org/C75630572","wikidata":"https://www.wikidata.org/wiki/Q538904","display_name":"Applied psychology","level":1,"score":0.3427000045776367},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3402000069618225},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.3190000057220459},{"id":"https://openalex.org/C99508421","wikidata":"https://www.wikidata.org/wiki/Q2678675","display_name":"Physical medicine and rehabilitation","level":1,"score":0.3066999912261963},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2840999960899353},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.2605000138282776},{"id":"https://openalex.org/C58328972","wikidata":"https://www.wikidata.org/wiki/Q184609","display_name":"Expert system","level":2,"score":0.23510000109672546}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2026.clpsych-1.24","doi":"https://doi.org/10.18653/v1/2026.clpsych-1.24","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.clpsych-1.24","pdf_url":"https://aclanthology.org/2026.clpsych-1.24.pdf","source":{"id":"https://openalex.org/S7407087913","display_name":"Workshop on Computational Linguistics and Clinical Psychology (CLPsych)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th Workshop on Computational Linguistics and Clinical Psychology (CLPsych 2026)","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2026.clpsych-1.24","doi":"https://doi.org/10.18653/v1/2026.clpsych-1.24","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2026.clpsych-1.24","pdf_url":"https://aclanthology.org/2026.clpsych-1.24.pdf","source":{"id":"https://openalex.org/S7407087913","display_name":"Workshop on Computational Linguistics and Clinical Psychology (CLPsych)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 10th Workshop on Computational Linguistics and Clinical Psychology (CLPsych 2026)","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7166804539.pdf","grobid_xml":"https://content.openalex.org/works/W7166804539.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W7208836099","https://openalex.org/W7171712616","https://openalex.org/W4408537679","https://openalex.org/W7223252693","https://openalex.org/W7219532974","https://openalex.org/W4402684011","https://openalex.org/W4415192165","https://openalex.org/W7117484483","https://openalex.org/W7139932326","https://openalex.org/W7203726217"],"abstract_inverted_index":{"Automated":[0],"feedback":[1,20,29,41,53,81,117,144,153],"is":[2],"increasingly":[3],"cited":[4],"as":[5,55],"a":[6],"key":[7],"advantage":[8],"of":[9,17,27,80,142],"AI-based":[10],"psychotherapy":[11,36],"training,":[12],"yet":[13],"the":[14,70,139],"clinical":[15,47,98,106,121,146],"groundedness":[16],"LLMgenerated":[18],"supervisory":[19,28],"remains":[21],"unevaluated.We":[22],"present":[23],"an":[24,33,134],"expert":[25],"evaluation":[26],"generated":[30],"by":[31],"PRACTICE,":[32],"LLM-powered":[34],"open-ended":[35],"training":[37,107,147],"simulator,":[38],"across":[39],"21":[40],"instances":[42,82],"from":[43],"four":[44],"novice":[45],"trainees.Two":[46],"psychology":[48],"experts":[49],"independently":[50],"coded":[51],"167":[52,71],"propositions":[54],"Justified,":[56],"Unjustified,":[57],"or":[58],"Unsure.Inter-rater":[59],"reliability":[60],"was":[61],"near-perfect":[62],"(raw":[63],"agreement":[64],"=":[65,68],"98.2%;":[66],"\u03ba":[67],"0.902).Of":[69],"propositions,":[72],"149":[73],"(89.2%)":[74],"were":[75],"rated":[76],"Justified;":[77],"however,":[78],"52.4%":[79],"contained":[83],"at":[84],"least":[85],"one":[86],"non-justified":[87],"proposition,":[88],"and":[89,103,148],"qualitative":[90],"analysis":[91],"identified":[92],"three":[93],"recurring":[94],"failure":[95],"types:":[96],"unsupported":[97],"inference,":[99],"referential":[100],"grounding":[101],"error,":[102],"pragmatic":[104],"failure.In":[105],"contexts,":[108],"even":[109],"low":[110],"error":[111],"rates":[112],"carry":[113],"ethical":[114],"weight:":[115],"unjustified":[116],"risks":[118],"reinforcing":[119],"inappropriate":[120],"behaviors":[122],"in":[123,145],"trainees":[124],"that":[125],"can":[126],"be":[127],"transferred":[128],"to":[129],"real":[130],"practice.These":[131],"findings":[132],"provide":[133],"initial":[135],"empirical":[136],"basis":[137],"for":[138,150],"responsible":[140],"deployment":[141],"LLM-generated":[143],"call":[149],"traceable,":[151],"expert-auditable":[152],"architectures.":[154]},"counts_by_year":[],"updated_date":"2026-10-28T10:33:24.094632","created_date":"2026-07-02T00:00:00"}