{"id":"https://openalex.org/W4387968576","doi":"https://doi.org/10.1145/3629171","title":"On Item-Sampling Evaluation for Recommender System","display_name":"On Item-Sampling Evaluation for Recommender System","publication_year":2023,"publication_date":"2023-10-27","ids":{"openalex":"https://openalex.org/W4387968576","doi":"https://doi.org/10.1145/3629171"},"language":"en","primary_location":{"id":"doi:10.1145/3629171","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3629171","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100407418","display_name":"Dong Li","orcid":"https://orcid.org/0000-0002-2599-6065"},"institutions":[{"id":"https://openalex.org/I149910238","display_name":"Kent State University","ror":"https://ror.org/049pfb863","country_code":"US","type":"education","lineage":["https://openalex.org/I149910238"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Dong Li","raw_affiliation_strings":["Kent State University, Kent, USA"],"raw_orcid":"https://orcid.org/0000-0002-2599-6065","affiliations":[{"raw_affiliation_string":"Kent State University, Kent, USA","institution_ids":["https://openalex.org/I149910238"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103270436","display_name":"Ruoming Jin","orcid":"https://orcid.org/0000-0003-1895-4243"},"institutions":[{"id":"https://openalex.org/I149910238","display_name":"Kent State University","ror":"https://ror.org/049pfb863","country_code":"US","type":"education","lineage":["https://openalex.org/I149910238"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruoming Jin","raw_affiliation_strings":["Kent State University, Kent, USA"],"raw_orcid":"https://orcid.org/0000-0003-1895-4243","affiliations":[{"raw_affiliation_string":"Kent State University, Kent, USA","institution_ids":["https://openalex.org/I149910238"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101738056","display_name":"Zhenming Liu","orcid":"https://orcid.org/0000-0001-9494-8748"},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]},{"id":"https://openalex.org/I267592682","display_name":"Williams (United States)","ror":"https://ror.org/007zhvp17","country_code":"US","type":"company","lineage":["https://openalex.org/I267592682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhenming Liu","raw_affiliation_strings":["College of William & Mary, Williamsburg, USA","iLambda Inc., Aurora, USA"],"raw_orcid":"https://orcid.org/0000-0001-9494-8748","affiliations":[{"raw_affiliation_string":"College of William & Mary, Williamsburg, USA","institution_ids":["https://openalex.org/I16285277","https://openalex.org/I267592682"]},{"raw_affiliation_string":"iLambda Inc., Aurora, USA","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058025888","display_name":"Bin Ren","orcid":"https://orcid.org/0000-0002-4116-5237"},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]},{"id":"https://openalex.org/I267592682","display_name":"Williams (United States)","ror":"https://ror.org/007zhvp17","country_code":"US","type":"company","lineage":["https://openalex.org/I267592682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bin Ren","raw_affiliation_strings":["College of William & Mary, Williamsburg, USA"],"raw_orcid":"https://orcid.org/0000-0002-4116-5237","affiliations":[{"raw_affiliation_string":"College of William & Mary, Williamsburg, USA","institution_ids":["https://openalex.org/I16285277","https://openalex.org/I267592682"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101864158","display_name":"Jing Gao","orcid":"https://orcid.org/0009-0002-4502-5650"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jing Gao","raw_affiliation_strings":["iLambda Inc., Aurora, USA"],"raw_orcid":"https://orcid.org/0009-0002-4502-5650","affiliations":[{"raw_affiliation_string":"iLambda Inc., Aurora, USA","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100382285","display_name":"Zhi Liu","orcid":"https://orcid.org/0000-0002-5248-4807"},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]},{"id":"https://openalex.org/I267592682","display_name":"Williams (United States)","ror":"https://ror.org/007zhvp17","country_code":"US","type":"company","lineage":["https://openalex.org/I267592682"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhi Liu","raw_affiliation_strings":["College of William & Mary, Williamsburg, USA","iLambda Inc., Aurora, USA"],"raw_orcid":"https://orcid.org/0000-0002-5248-4807","affiliations":[{"raw_affiliation_string":"College of William & Mary, Williamsburg, USA","institution_ids":["https://openalex.org/I16285277","https://openalex.org/I267592682"]},{"raw_affiliation_string":"iLambda Inc., Aurora, USA","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100407418"],"corresponding_institution_ids":["https://openalex.org/I149910238"],"apc_list":null,"apc_paid":null,"fwci":2.6905,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.9198331,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"2","issue":"1","first_page":"1","last_page":"36"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9876999855041504,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8233610987663269},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7214609980583191},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.676771342754364},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.6721213459968567},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5767489671707153},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5396120548248291},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5071938037872314},{"id":"https://openalex.org/keywords/collaborative-filtering","display_name":"Collaborative filtering","score":0.504226803779602},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.4894470274448395},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.48894253373146057},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.488706111907959},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4445529878139496},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.42547935247421265},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3700735569000244},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.08318307995796204}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8233610987663269},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7214609980583191},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.676771342754364},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.6721213459968567},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5767489671707153},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5396120548248291},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5071938037872314},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.504226803779602},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.4894470274448395},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.48894253373146057},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.488706111907959},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4445529878139496},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.42547935247421265},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3700735569000244},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.08318307995796204},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C114614502","wikidata":"https://www.wikidata.org/wiki/Q76592","display_name":"Combinatorics","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3629171","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3629171","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/17","display_name":"Partnerships for the goals","score":0.44999998807907104}],"awards":[{"id":"https://openalex.org/G3000011090","display_name":"EAGER: Collaborative Research: On the Theoretical Foundation of Recommendation System Evaluation","funder_award_id":"2142675","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320311890","display_name":"Kent State University","ror":"https://ror.org/049pfb863"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1994389483","https://openalex.org/W2072271719","https://openalex.org/W2082029016","https://openalex.org/W2114079787","https://openalex.org/W2118934678","https://openalex.org/W2142144955","https://openalex.org/W2150886314","https://openalex.org/W2241862190","https://openalex.org/W2798972759","https://openalex.org/W2884134047","https://openalex.org/W2892888989","https://openalex.org/W2903296288","https://openalex.org/W2912745432","https://openalex.org/W2913491198","https://openalex.org/W2963085847","https://openalex.org/W2993053728","https://openalex.org/W2997546679","https://openalex.org/W2998206837","https://openalex.org/W3080781457","https://openalex.org/W3097103183","https://openalex.org/W3097991661","https://openalex.org/W3101708421","https://openalex.org/W3135330817","https://openalex.org/W3164446158","https://openalex.org/W3176338027","https://openalex.org/W3185773347","https://openalex.org/W3201149665","https://openalex.org/W3201258060","https://openalex.org/W4229012946","https://openalex.org/W4234386980","https://openalex.org/W4250589301","https://openalex.org/W4301459447","https://openalex.org/W6891924792"],"related_works":["https://openalex.org/W2772628444","https://openalex.org/W1484355083","https://openalex.org/W4220714703","https://openalex.org/W2098758514","https://openalex.org/W2735929803","https://openalex.org/W3008845055","https://openalex.org/W2170391450","https://openalex.org/W4376854386","https://openalex.org/W2202724490","https://openalex.org/W2508671622"],"abstract_inverted_index":{"Personalized":[0],"recommender":[1,133],"systems":[2],"play":[3],"a":[4,69,100,151,177,207,228,264,317,323,354,384,392],"crucial":[5],"role":[6],"in":[7,11,210,253,347],"modern":[8],"society,":[9],"especially":[10],"e-commerce,":[12],"news,":[13],"and":[14,19,44,87,121,189,195,236,305,390],"ads":[15],"areas.":[16],"Correctly":[17],"evaluating":[18,45],"comparing":[20],"candidate":[21],"recommendation":[22,49,72,102],"models":[23,50,73],"is":[24,35,94,166,202,286,350],"as":[25,27],"essential":[26],"constructing":[28],"ones.":[29],"The":[30],"common":[31],"offline":[32,213],"evaluation":[33,97,169,184,201,209,235,349,377,389],"strategy":[34],"holding":[36],"out":[37],"some":[38,117],"user-interacted":[39],"items":[40,55,140,156],"from":[41],"training":[42],"data":[43],"the":[46,71,77,80,85,114,129,132,138,160,163,199,223,231,241,256,269,273,278,282,288,291,301,309,328,334,361,374,400,403],"performance":[47,161],"of":[48,131,136,155,162,180,212,255,268,277,290,303,327,381,402],"based":[51],"on":[52,113],"how":[53],"many":[54],"they":[56],"can":[57,123,294],"retrieve.":[58],"Specifically,":[59],"for":[60,68,342,387],"any":[61],"hold-out":[62],"item":[63,67,86],"or":[64],"so-called":[65],"target":[66,139],"user,":[70],"try":[74],"to":[75,108,126,147,186,206,249,364],"predict":[76],"probability":[78],"that":[79,198,359],"user":[81,375],"would":[82,104],"interact":[83],"with":[84,204,247,311],"rank":[88,148,275,284],"it":[89],"among":[90,141,150],"overall":[91],"items,":[92,143],"which":[93,165],"called":[95,167],"global":[96,208,237,251,274,292],".":[98,170],"Intuitively,":[99],"good":[101],"model":[103],"assign":[105],"high":[106],"probabilities":[107],"such":[109],"hold-out/target":[110],"items.":[111],"Based":[112],"specific":[115],"ranks,":[116],"metrics":[118,341],"like":[119],"Recall@K":[120],"NDCG@K":[122],"be":[124,295],"calculated":[125],"further":[127,296],"quantify":[128],"quality":[130],"model.":[134],"Instead":[135],"ranking":[137],"all":[142],"Koren":[144],"first":[145,221],"proposed":[146,329],"them":[149],"small":[152,343],"sampled":[153],"set":[154],",":[157],"then":[158],"quantified":[159],"models,":[164],"sampling":[168,183,200,234,246,270,348,357,376,388,404],"Ever":[171],"since":[172],"then,":[173],"there":[174],"has":[175],"been":[176],"large":[178],"amount":[179],"work":[181,302],"adopting":[182],"due":[185],"its":[187,250],"efficiency":[188],"frugality.":[190],"In":[191,217],"recent":[192],"work,":[193,219],"Rendle":[194],"Krichene":[196,304],"argued":[197],"\u201cinconsistent\u201d":[203,224],"respect":[205,248],"terms":[211,254],"top-":[214,257,344],"K":[215,258,345],"metrics.":[216],"this":[218,365],"we":[220,262,299,352,371],"investigate":[222],"phenomenon":[225],"by":[226],"taking":[227],"glance":[229],"at":[230],"connections":[232],"between":[233,245],"evaluation.":[238],"We":[239],"reveal":[240],"approximately":[242],"linear":[243],"relationship":[244],"counterpart":[252],"Recall":[259],"metric.":[260],"Second,":[261],"propose":[263,353],"new":[265],"statistical":[266],"perspective":[267],"evaluation\u2014to":[271],"estimate":[272],"distribution":[276,285],"entire":[279],"population.":[280],"After":[281],"estimated":[283],"obtained,":[287],"approximation":[289],"metric":[293,330],"derived.":[297],"Third,":[298],"extend":[300],"Rendle,":[306],"directly":[307],"optimizing":[308],"error":[310],"ground":[312],"truth,":[313],"providing":[314],"not":[315,369],"only":[316],"comprehensive":[318],"empirical":[319,397],"study":[320,373],"but":[321,368],"also":[322,372],"rigorous":[324],"theoretical":[325,394],"understanding":[326],"estimators.":[331],"To":[332],"address":[333],"\u201cblind":[335],"spot\u201d":[336],"issue,":[337],"where":[338],"accurately":[339],"estimating":[340],"values":[346],"challenging,":[351],"novel":[355],"adaptive":[356],"method":[358],"generalizes":[360],"expectation-maximization":[362],"algorithm":[363],"setting.":[366],"Last":[367],"least,":[370],"effect.":[378],"This":[379],"series":[380],"works":[382],"outlines":[383],"clear":[385],"roadmap":[386],"establishes":[391],"foundational":[393],"framework.":[395],"Extensive":[396],"studies":[398],"validate":[399],"reliability":[401],"methods":[405],"presented.":[406]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}