{"id":"https://openalex.org/W7165658285","doi":"https://doi.org/10.48550/arxiv.2606.22961","title":"LLM-as-a-Judge for Reliable and Explainable Offline Evaluation in Top-K Recommendation","display_name":"LLM-as-a-Judge for Reliable and Explainable Offline Evaluation in Top-K Recommendation","publication_year":2026,"publication_date":"2026-06-22","ids":{"openalex":"https://openalex.org/W7165658285","doi":"https://doi.org/10.48550/arxiv.2606.22961"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.22961","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.22961","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139184593","display_name":"Yue Que","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Que, Yue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139180848","display_name":"Junyi Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Junyi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139182494","display_name":"Xiaokun Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Xiaokun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139158261","display_name":"Haiming Jin","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Haiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100926972","display_name":"Qiao Xiang","orcid":"https://orcid.org/0000-0002-2696-9835"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiang, Qiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139191725","display_name":"Chen Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.8848999738693237,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.8848999738693237,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.057500001043081284,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.00559999980032444,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.7069000005722046},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6150000095367432},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.5914000272750854},{"id":"https://openalex.org/keywords/matching","display_name":"Matching (statistics)","score":0.5406000018119812},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.4941999912261963},{"id":"https://openalex.org/keywords/semantic-matching","display_name":"Semantic matching","score":0.48010000586509705},{"id":"https://openalex.org/keywords/crowdsourcing","display_name":"Crowdsourcing","score":0.4250999987125397},{"id":"https://openalex.org/keywords/relevance","display_name":"Relevance (law)","score":0.4050000011920929}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8054999709129333},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.7069000005722046},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6150000095367432},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.5914000272750854},{"id":"https://openalex.org/C165064840","wikidata":"https://www.wikidata.org/wiki/Q1321061","display_name":"Matching (statistics)","level":2,"score":0.5406000018119812},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.4941999912261963},{"id":"https://openalex.org/C2778493491","wikidata":"https://www.wikidata.org/wiki/Q7449072","display_name":"Semantic matching","level":3,"score":0.48010000586509705},{"id":"https://openalex.org/C62230096","wikidata":"https://www.wikidata.org/wiki/Q275969","display_name":"Crowdsourcing","level":2,"score":0.4250999987125397},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.4050000011920929},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3986999988555908},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3880999982357025},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.3596000075340271},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.35249999165534973},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.31709998846054077},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3133000135421753},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.30410000681877136},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.2971000075340271},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.29510000348091125},{"id":"https://openalex.org/C2779532271","wikidata":"https://www.wikidata.org/wiki/Q445558","display_name":"Relevance feedback","level":4,"score":0.2791999876499176},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C4679612","wikidata":"https://www.wikidata.org/wiki/Q866298","display_name":"Aggregate (composite)","level":2,"score":0.2583000063896179},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.2581999897956848}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.22961","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.22961","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.22961","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recommendation":[0],"evaluation":[1,21,49],"plays":[2],"a":[3,55,117,132,173],"crucial":[4],"role":[5],"in":[6,155,221],"guiding":[7],"the":[8,66,106,156,163,169,187,212],"refinement":[9],"and":[10,43,60,68,79,86,119,153,199,219],"deployment":[11],"of":[12,57,109],"recommender":[13],"systems.":[14],"Most":[15],"existing":[16],"trials":[17],"rely":[18],"on":[19,162],"offline":[20,48,110,124],"using":[22],"Top-K":[23,91,192],"metrics":[24,92,193],"computed":[25],"over":[26],"holdout":[27,164],"user":[28,52,136],"behaviors.":[29],"However,":[30],"we":[31,115,130,185],"identify":[32],"two":[33],"fundamental":[34],"limitations":[35],"that":[36,211],"undermine":[37],"their":[38,141],"ability":[39],"to":[40,84,101,139,176,194],"deliver":[41],"reliable":[42,118],"explainable":[44,120],"evaluations.":[45],"Regarding":[46,89],"reliability,":[47,129,217],"treats":[50],"observed":[51],"feedback":[53,72],"as":[54],"proxy":[56,67,134,145],"true":[58,142],"preferences":[59,152],"enforces":[61],"rigid":[62],"ID":[63],"matching":[64,150],"between":[65,151],"recommendation.":[69],"In":[70,112],"practice,":[71],"collections":[73],"are":[74],"inherently":[75],"shaped":[76],"by":[77],"incomplete":[78],"biased":[80],"item":[81],"exposure,":[82],"leading":[83],"distorted":[85],"unreliable":[87],"assessments.":[88],"explainability,":[90,168,218],"only":[93],"establish":[94],"numerical":[95],"scores":[96,189],"without":[97],"offering":[98],"meaningful":[99],"insights":[100],"support":[102],"them,":[103],"thereby":[104],"reinforcing":[105],"black-box":[107],"nature":[108],"evaluation.":[111,126,222],"this":[113],"paper,":[114],"propose":[116],"LLM-as-a-Judge":[121],"framework":[122],"for":[123,147,202],"recommendation":[125,197],"To":[127,166],"enhance":[128],"introduce":[131],"semantic":[133,157],"from":[135],"textual":[137],"behaviors":[138],"represent":[140],"preferences.":[143],"This":[144],"allows":[146],"more":[148],"flexible":[149],"recommendations":[154],"space,":[158],"rather":[159],"than":[160],"depending":[161],"feedback.":[165],"ensure":[167],"LLM":[170,213],"Judge":[171,214],"adopts":[172],"reasoning-then-scoring":[174],"process":[175],"generate":[177],"relevance":[178],"judgments":[179],"along":[180],"with":[181],"explicit":[182],"rationale.":[183],"Finally,":[184],"aggregate":[186],"individual":[188],"into":[190],"global":[191],"quantify":[195],"overall":[196],"quality,":[198],"provide":[200],"justification":[201],"each":[203],"preference":[204],"hit":[205],"or":[206],"miss.":[207],"Extensive":[208],"experiments":[209],"demonstrate":[210],"achieves":[215],"solid":[216],"robustness":[220]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-24T00:00:00"}