{"id":"https://openalex.org/W7167015144","doi":"https://doi.org/10.1145/3786579.3804927","title":"Evaluating the Evaluator: LLM Evaluators for Patient-Facing AI Agents","display_name":"Evaluating the Evaluator: LLM Evaluators for Patient-Facing AI Agents","publication_year":2026,"publication_date":"2026-07-02","ids":{"openalex":"https://openalex.org/W7167015144","doi":"https://doi.org/10.1145/3786579.3804927"},"language":null,"primary_location":{"id":"doi:10.1145/3786579.3804927","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786579.3804927","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Interactive Health Conference","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3786579.3804927","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024945104","display_name":"Angela Mastrianni","orcid":"https://orcid.org/0000-0001-8179-0101"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Angela Mastrianni","raw_affiliation_strings":["Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0001-8179-0101","affiliations":[{"raw_affiliation_string":"Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057953305","display_name":"Katerina Andreadis","orcid":"https://orcid.org/0000-0001-8586-450X"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Katerina Andreadis","raw_affiliation_strings":["Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0001-8586-450X","affiliations":[{"raw_affiliation_string":"Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101909563","display_name":"Ji Chen","orcid":"https://orcid.org/0000-0002-4856-0141"},"institutions":[{"id":"https://openalex.org/I4210086933","display_name":"NYU Langone Health","ror":"https://ror.org/005dvqh91","country_code":"US","type":"healthcare","lineage":["https://openalex.org/I4210086933"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ji Chen","raw_affiliation_strings":["Medical Center Information Technology, NYU Langone Health, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-4856-0141","affiliations":[{"raw_affiliation_string":"Medical Center Information Technology, NYU Langone Health, New York, New York, USA","institution_ids":["https://openalex.org/I4210086933"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100739017","display_name":"Danissa V. Rodriguez","orcid":"https://orcid.org/0000-0003-4642-6798"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Danissa V Rodriguez","raw_affiliation_strings":["Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0003-4642-6798","affiliations":[{"raw_affiliation_string":"Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007631989","display_name":"David Mann","orcid":"https://orcid.org/0000-0002-2099-0852"},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Devin Mann","raw_affiliation_strings":["Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA"],"raw_orcid":"https://orcid.org/0000-0002-2099-0852","affiliations":[{"raw_affiliation_string":"Department of Population Health, NYU Grossman School of Medicine, New York, New York, USA and Department of Health Informatics, NYU Langone Health, New York, New York, USA","institution_ids":["https://openalex.org/I57206974"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.86504723,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"6"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5142999887466431,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},"topics":[{"id":"https://openalex.org/T11636","display_name":"Artificial Intelligence in Healthcare and Education","score":0.5142999887466431,"subfield":{"id":"https://openalex.org/subfields/2718","display_name":"Health Informatics"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.1046999990940094,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.08299999684095383,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/data-collection","display_name":"Data collection","score":0.271699994802475},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.2524000108242035},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.24779999256134033},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.24539999663829803},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.24369999766349792}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4542999863624573},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3197999894618988},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.271699994802475},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2524000108242035},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.24779999256134033},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.24539999663829803},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.24369999766349792},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.2312999963760376},{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.22429999709129333},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.2206999957561493}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3786579.3804927","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786579.3804927","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Interactive Health Conference","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3786579.3804927","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3786579.3804927","pdf_url":null,"source":null,"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2026 ACM Interactive Health Conference","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":19,"referenced_works":["https://openalex.org/W1563373471","https://openalex.org/W2050361346","https://openalex.org/W2763766763","https://openalex.org/W3205928582","https://openalex.org/W4389519254","https://openalex.org/W4396938113","https://openalex.org/W4401754043","https://openalex.org/W4401824301","https://openalex.org/W4402683144","https://openalex.org/W4408615073","https://openalex.org/W4408863244","https://openalex.org/W4412888095","https://openalex.org/W4412889096","https://openalex.org/W4415257614","https://openalex.org/W4415930269","https://openalex.org/W4416273060","https://openalex.org/W7124460067","https://openalex.org/W7131904554","https://openalex.org/W7133239970"],"related_works":[],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2],"(LLM)":[3],"evaluators":[4],"have":[5],"been":[6],"proposed":[7],"as":[8],"a":[9,39,48],"scalable":[10],"supplement":[11],"to":[12,98],"human":[13,32],"review":[14],"of":[15,26,42,59,65,88,102,117],"AI-generated":[16],"content":[17],"in":[18,121,135],"clinical":[19,122],"settings.":[20],"To":[21],"be":[22],"effective,":[23],"the":[24,54,60,85,89,100,103,113,130],"outputs":[25],"LLM":[27,44,61,73,90,104,118],"evaluation":[28,41,55],"must":[29],"align":[30],"with":[31,63,77],"judgment.":[33],"In":[34],"this":[35,110,136],"paper,":[36],"we":[37,95],"present":[38],"meta":[40],"an":[43],"evaluator":[45,62,119],"developed":[46],"for":[47,126],"patient-facing":[49],"conversation":[50],"agent.":[51],"We":[52,106],"compared":[53],"process":[56],"and":[57,74,124,132],"outcomes":[58],"those":[64],"five":[66],"clinicians.":[67],"Our":[68],"findings":[69],"highlight":[70],"discrepancies":[71],"between":[72],"clinician":[75],"evaluation,":[76],"clinicians":[78],"considering":[79],"certain":[80],"criteria":[81],"not":[82],"captured":[83],"by":[84],"current":[86],"version":[87],"evaluator.":[91,105],"From":[92],"our":[93],"findings,":[94],"identify":[96],"opportunities":[97],"improve":[99],"design":[101],"also":[107],"discuss":[108],"how":[109],"meta-evaluation":[111],"highlights":[112],"inherently":[114],"sociotechnical":[115],"nature":[116],"development":[120],"settings,":[123],"argue":[125],"greater":[127],"involvement":[128],"from":[129],"HCI":[131],"CSCW":[133],"communities":[134],"research":[137],"area.":[138]},"counts_by_year":[],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2026-07-03T00:00:00"}