{"id":"https://openalex.org/W7207601463","doi":"https://doi.org/10.1145/3773078.3831835","title":"The Utility of LLMs in Recommender Systems Explanation Evaluation","display_name":"The Utility of LLMs in Recommender Systems Explanation Evaluation","alternate_titles":[],"publication_year":2026,"publication_date":"2026-09-25","ids":{"openalex":"https://openalex.org/W7207601463","doi":"https://doi.org/10.1145/3773078.3831835"},"language":"en","primary_location":{"id":"doi:10.1145/3773078.3831835","doi":"https://doi.org/10.1145/3773078.3831835","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773078.3831835","pdf_url":null,"source":{"id":"https://openalex.org/S7407086088","display_name":"ACM Conference on Recommender Systems (RecSys)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W7207601463"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3773078.3831835","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028528710","display_name":"Kathrin Wardatzky","orcid":"https://orcid.org/0000-0002-7043-7326","observed_orcids":["https://orcid.org/0000-0002-7043-7326"]},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Kathrin Wardatzky","raw_affiliation_strings":["Department of Informatics, University of Zurich, Zurich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-7043-7326","affiliations":[{"raw_affiliation_string":"Department of Informatics, University of Zurich, Zurich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5056796427","display_name":"Oana Inel","orcid":"https://orcid.org/0000-0003-4691-6586","observed_orcids":["https://orcid.org/0000-0003-4691-6586"]},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Oana Inel","raw_affiliation_strings":["Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0003-4691-6586","affiliations":[{"raw_affiliation_string":"Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052015967","display_name":"Luca Rossetto","orcid":"https://orcid.org/0000-0002-5389-9465","observed_orcids":["https://orcid.org/0000-0002-5389-9465"]},"institutions":[{"id":"https://openalex.org/I42934936","display_name":"Dublin City University","ror":"https://ror.org/04a1a1e81","country_code":"IE","type":"education","lineage":["https://openalex.org/I42934936"]}],"countries":["IE"],"is_corresponding":false,"raw_author_name":"Luca Rossetto","raw_affiliation_strings":["School of Computing, Dublin City University, Dublin, Ireland"],"raw_orcid":"https://orcid.org/0000-0002-5389-9465","affiliations":[{"raw_affiliation_string":"School of Computing, Dublin City University, Dublin, Ireland","institution_ids":["https://openalex.org/I42934936"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5073592405","display_name":"Abraham Bernstein","orcid":"https://orcid.org/0000-0002-0128-4602","observed_orcids":["https://orcid.org/0000-0002-0128-4602"]},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Abraham Bernstein","raw_affiliation_strings":["University of Zurich, Zurich, Switzerland"],"raw_orcid":"https://orcid.org/0000-0002-0128-4602","affiliations":[{"raw_affiliation_string":"University of Zurich, Zurich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.7694,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.85692334,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"534","last_page":"543"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.8521000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.8521000146865845,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.14329999685287476,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.0005000000237487257,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-systems","display_name":"recommender systems","score":0.8679999709129333},{"id":"https://openalex.org/keywords/evaluation-of-explanations","display_name":"explanation evaluation","score":0.8450000286102295},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8700000047683716},{"id":"https://openalex.org/keywords/automatic-evaluation-metrics","display_name":"automatic evaluation metrics","score":0.6800000071525574},{"id":"https://openalex.org/keywords/explanation-selection","display_name":"explanation selection","score":0.4129999876022339},{"id":"https://openalex.org/keywords/human-evaluation","display_name":"human evaluation","score":0.3050000071525574},{"id":"https://openalex.org/keywords/llm-as-a-judge","display_name":"LLM-as-a-judge","score":0.4909999966621399},{"id":"https://openalex.org/keywords/prompt-engineering","display_name":"prompt engineering","score":0.6660000085830688},{"id":"https://openalex.org/keywords/model-size-comparison","display_name":"model size comparison","score":0.5929999947547913},{"id":"https://openalex.org/keywords/temperature-settings","display_name":"setting temperature","score":0.7139999866485596}],"concepts":[{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.6051999926567078},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.572700023651123},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.545199990272522},{"id":"https://openalex.org/C199521495","wikidata":"https://www.wikidata.org/wiki/Q181487","display_name":"Audit","level":2,"score":0.538100004196167},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.42489999532699585},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.42179998755455017},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38429999351501465},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3626999855041504},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3515999913215637},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.31679999828338623},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.3165000081062317},{"id":"https://openalex.org/C88006597","wikidata":"https://www.wikidata.org/wiki/Q690117","display_name":"Disk formatting","level":2,"score":0.3082999885082245},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.298799991607666},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.29280000925064087},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.27639999985694885},{"id":"https://openalex.org/C42475967","wikidata":"https://www.wikidata.org/wiki/Q194292","display_name":"Operations research","level":1,"score":0.2702000141143799},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.2700999975204468},{"id":"https://openalex.org/C26760741","wikidata":"https://www.wikidata.org/wiki/Q160402","display_name":"Perception","level":2,"score":0.2572999894618988}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1145/3773078.3831835","doi":"https://doi.org/10.1145/3773078.3831835","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773078.3831835","pdf_url":null,"source":{"id":"https://openalex.org/S7407086088","display_name":"ACM Conference on Recommender Systems (RecSys)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2609.01627","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.01627","pdf_url":"https://arxiv.org/pdf/2609.01627","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:www.zora.uzh.ch:20.500.14742/250178","doi":null,"is_oa":true,"landing_page_url":"https://www.zora.uzh.ch/handle/20.500.14742/250178","pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Wardatzky, K., Inel, O., Rossetto, L., & Bernstein, A. (2026). The Utility of LLMs in Recommender Systems Explanation Evaluation (G. Adomavicius, G. Karypis, J. Konstan, & et al, Eds.; No. 20; pp. 534\u2013543). ACM Digital library. https://doi.org/10.1145/3773078.3831835","raw_type":"Conference or Workshop Item"},{"id":"doi:10.48550/arxiv.2609.01627","doi":"https://doi.org/10.48550/arxiv.2609.01627","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.01627","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"doi:10.5167/uzh-436484","doi":"https://doi.org/10.5167/uzh-436484","is_oa":true,"landing_page_url":"https://doi.org/10.5167/uzh-436484","pdf_url":null,"source":{"id":"https://openalex.org/S4306401281","display_name":"Zurich Open Repository and Archive (University of Zurich)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I202697423","host_organization_name":"University of Zurich","host_organization_lineage":["https://openalex.org/I202697423"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"ConferencePaper"}],"best_oa_location":{"id":"doi:10.1145/3773078.3831835","doi":"https://doi.org/10.1145/3773078.3831835","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3773078.3831835","pdf_url":null,"source":{"id":"https://openalex.org/S7407086088","display_name":"ACM Conference on Recommender Systems (RecSys)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 20th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7207601463.pdf","grobid_xml":"https://content.openalex.org/works/W7207601463.grobid-xml"},"referenced_works_count":45,"referenced_works":["https://openalex.org/W2100171245","https://openalex.org/W2101105183","https://openalex.org/W2127795553","https://openalex.org/W2141960018","https://openalex.org/W2154652894","https://openalex.org/W2223821166","https://openalex.org/W2670253439","https://openalex.org/W2807246965","https://openalex.org/W2905901805","https://openalex.org/W2917259853","https://openalex.org/W2950275995","https://openalex.org/W2954870332","https://openalex.org/W3034491755","https://openalex.org/W3093532296","https://openalex.org/W3093963954","https://openalex.org/W3094497946","https://openalex.org/W3195311662","https://openalex.org/W4200575860","https://openalex.org/W4205777432","https://openalex.org/W4212975546","https://openalex.org/W4224936585","https://openalex.org/W4309765757","https://openalex.org/W4386431357","https://openalex.org/W4386730597","https://openalex.org/W4389564102","https://openalex.org/W4401798709","https://openalex.org/W4401864369","https://openalex.org/W4402671800","https://openalex.org/W4403220697","https://openalex.org/W4403220991","https://openalex.org/W4405347090","https://openalex.org/W4406849765","https://openalex.org/W4407184412","https://openalex.org/W4407592733","https://openalex.org/W4411120594","https://openalex.org/W4415382061","https://openalex.org/W4415797148","https://openalex.org/W4416017051","https://openalex.org/W4416018129","https://openalex.org/W4417113720","https://openalex.org/W7106264473","https://openalex.org/W7108224864","https://openalex.org/W7112902589","https://openalex.org/W7119510601","https://openalex.org/W7136040605"],"related_works":["https://openalex.org/W7160819649","https://openalex.org/W4403220991","https://openalex.org/W4400106426","https://openalex.org/W7140908655","https://openalex.org/W4406849765","https://openalex.org/W4416017051","https://openalex.org/W4405347090","https://openalex.org/W7221519308","https://openalex.org/W7162666156","https://openalex.org/W7215087369"],"abstract_inverted_index":{"Explanations":[0],"play":[1],"a":[2,13,50,82,128,178],"crucial":[3],"role":[4],"in":[5,121],"creating":[6],"trustworthy":[7],"recommender":[8],"systems":[9],"(RS),":[10],"yet":[11,110],"choosing":[12],"good":[14],"explanation":[15,20,35,103,125,138,154],"method":[16,126],"presents":[17],"challenges.":[18],"Many":[19],"methods":[21,37],"exist,":[22],"but":[23,105],"little":[24],"guidance":[25],"exists":[26],"on":[27],"which":[28,32,85,156],"is":[29,63,86,203],"best":[30],"for":[31,102,127,226,234],"setting.":[33],"Existing":[34],"generation":[36],"often":[38,70],"produce":[39],"abstract":[40,76],"outputs":[41],"that":[42,93,184],"require":[43,79],"further":[44],"formatting":[45],"to":[46,136],"become":[47],"user-friendly,":[48],"with":[49,81,196],"seemingly":[51],"endless":[52],"pool":[53],"of":[54,59,119,163],"options.":[55],"Running":[56],"user-based":[57],"evaluations":[58],"all":[60],"possible":[61],"options":[62],"usually":[64],"unfeasible,":[65],"while":[66,185],"automated":[67],"evaluation":[68,212,229],"metrics":[69],"either":[71],"assess":[72],"only":[73],"the":[74,117,144,147],"explainer\u2019s":[75],"output":[77],"or":[78],"comparison":[80],"ground":[83],"truth,":[84],"generally":[87],"unavailable.":[88],"Recent":[89],"studies":[90,116],"have":[91],"shown":[92],"large":[94],"language":[95],"models":[96,225],"(LLMs)":[97],"can":[98],"serve":[99],"as":[100,237],"\u201cjudges\u201d":[101],"evaluation,":[104,227],"their":[106,134,199],"reliability":[107],"has":[108],"not":[109],"been":[111],"thoroughly":[112],"explored.":[113],"This":[114],"paper":[115],"utility":[118],"LLMs":[120,162,186,241],"selecting":[122],"an":[123],"effective":[124],"given":[129,140],"application.":[130],"We":[131,170,214],"first":[132],"explore":[133],"ability":[135],"generate":[137,151],"prototypes":[139],"varying":[141,164],"information":[142],"about":[143],"RS":[145],"and":[146,191,205,211,231],"user.":[148],"Specifically,":[149],"we":[150],"18":[152],"distinct":[153],"prototypes,":[155],"are":[157],"subsequently":[158],"evaluated":[159],"by":[160,208],"14":[161],"sizes":[165],"across":[166],"two":[167],"temperature":[168],"settings.":[169],"compare":[171],"these":[172],"against":[173],"human":[174,197],"ratings":[175],"derived":[176],"from":[177],"user":[179],"study.":[180],"Our":[181],"results":[182],"show":[183],"exhibit":[187],"human-like":[188],"rating":[189,201],"patterns":[190],"achieve":[192],"moderate":[193],"rank":[194],"correlation":[195],"raters,":[198],"absolute":[200],"agreement":[202],"low":[204],"varies":[206],"substantially":[207],"model":[209],"size":[210],"construct.":[213],"derive":[215],"four":[216],"practical":[217],"recommendations:":[218],"keep":[219],"explanation-generation":[220],"prompts":[221],"concise,":[222],"prefer":[223],"larger":[224],"pre-test":[228],"constructs,":[230],"audit":[232],"explanations":[233],"factual":[235],"accuracy,":[236],"neither":[238],"humans":[239],"nor":[240],"reliably":[242],"detect":[243],"non-factual":[244],"content.":[245]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7207601463","counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-09-04T00:00:00"}
