{"id":"https://openalex.org/W3154199194","doi":"https://doi.org/10.1145/3569930","title":"A Critical Study on Data Leakage in Recommender System Offline Evaluation","display_name":"A Critical Study on Data Leakage in Recommender System Offline Evaluation","publication_year":2022,"publication_date":"2022-10-28","ids":{"openalex":"https://openalex.org/W3154199194","doi":"https://doi.org/10.1145/3569930","mag":"3154199194"},"language":"en","primary_location":{"id":"doi:10.1145/3569930","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3569930","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2010.11060","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5084274414","display_name":"Yitong Ji","orcid":"https://orcid.org/0000-0002-3210-3595"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yitong Ji","raw_affiliation_strings":["Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-3210-3595","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100618738","display_name":"Aixin Sun","orcid":"https://orcid.org/0000-0003-0764-4258"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Aixin Sun","raw_affiliation_strings":["Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0003-0764-4258","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100436659","display_name":"Jie Zhang","orcid":"https://orcid.org/0000-0001-8996-7581"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Jie Zhang","raw_affiliation_strings":["Nanyang Technological University, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-8996-7581","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100734069","display_name":"Chenliang Li","orcid":"https://orcid.org/0000-0003-3144-6374"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chenliang Li","raw_affiliation_strings":["Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0003-3144-6374","affiliations":[{"raw_affiliation_string":"Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":18.45,"has_fulltext":false,"cited_by_count":76,"citation_normalized_percentile":{"value":0.9927361,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":"41","issue":"3","first_page":"1","last_page":"27"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9904999732971191,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11165","display_name":"Image and Video Quality Assessment","score":0.9646999835968018,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/timeline","display_name":"Timeline","score":0.9322546124458313},{"id":"https://openalex.org/keywords/movielens","display_name":"MovieLens","score":0.9178769588470459},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8274911642074585},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7183706760406494},{"id":"https://openalex.org/keywords/collaborative-filtering","display_name":"Collaborative filtering","score":0.48506107926368713},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.47195783257484436},{"id":"https://openalex.org/keywords/leakage","display_name":"Leakage (economics)","score":0.43201401829719543},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3930446207523346},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.382951945066452}],"concepts":[{"id":"https://openalex.org/C4438859","wikidata":"https://www.wikidata.org/wiki/Q186117","display_name":"Timeline","level":2,"score":0.9322546124458313},{"id":"https://openalex.org/C2776156558","wikidata":"https://www.wikidata.org/wiki/Q4353746","display_name":"MovieLens","level":4,"score":0.9178769588470459},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8274911642074585},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7183706760406494},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.48506107926368713},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.47195783257484436},{"id":"https://openalex.org/C2777042071","wikidata":"https://www.wikidata.org/wiki/Q6509304","display_name":"Leakage (economics)","level":2,"score":0.43201401829719543},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3930446207523346},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.382951945066452},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C139719470","wikidata":"https://www.wikidata.org/wiki/Q39680","display_name":"Macroeconomics","level":1,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3569930","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3569930","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2010.11060","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.11060","pdf_url":"https://arxiv.org/pdf/2010.11060","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:dr.ntu.edu.sg:10356/170569","is_oa":false,"landing_page_url":"https://hdl.handle.net/10356/170569","pdf_url":null,"source":{"id":"https://openalex.org/S4306402609","display_name":"DR-NTU (Nanyang Technological University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I172675005","host_organization_name":"Nanyang Technological University","host_organization_lineage":["https://openalex.org/I172675005"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2010.11060","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2010.11060","pdf_url":"https://arxiv.org/pdf/2010.11060","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.5}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W298464722","https://openalex.org/W2008886893","https://openalex.org/W2072992969","https://openalex.org/W2078488685","https://openalex.org/W2097638479","https://openalex.org/W2104391728","https://openalex.org/W2134040744","https://openalex.org/W2140310134","https://openalex.org/W2153274408","https://openalex.org/W2160541537","https://openalex.org/W2605350416","https://openalex.org/W2736287575","https://openalex.org/W2739273093","https://openalex.org/W2753074643","https://openalex.org/W2798385737","https://openalex.org/W2892821876","https://openalex.org/W2893427712","https://openalex.org/W2903773361","https://openalex.org/W2908054697","https://openalex.org/W2908404712","https://openalex.org/W2940934175","https://openalex.org/W2950416221","https://openalex.org/W2954765895","https://openalex.org/W2963367478","https://openalex.org/W2963655167","https://openalex.org/W2964044287","https://openalex.org/W2966349618","https://openalex.org/W2971196067","https://openalex.org/W2972520532","https://openalex.org/W2993053728","https://openalex.org/W2994700598","https://openalex.org/W2997261254","https://openalex.org/W2997477628","https://openalex.org/W2998206837","https://openalex.org/W3011685549","https://openalex.org/W3032861513","https://openalex.org/W3034718811","https://openalex.org/W3035178789","https://openalex.org/W3035666843","https://openalex.org/W3045200674","https://openalex.org/W3081170586","https://openalex.org/W3082291914","https://openalex.org/W3088444111","https://openalex.org/W3088694469","https://openalex.org/W3102755086","https://openalex.org/W3105114834","https://openalex.org/W3105297929","https://openalex.org/W3122507327","https://openalex.org/W3134063549","https://openalex.org/W3154190849","https://openalex.org/W3156939347","https://openalex.org/W3198076800","https://openalex.org/W3199744737","https://openalex.org/W3208227120","https://openalex.org/W4212843742","https://openalex.org/W4288080156","https://openalex.org/W4292092796"],"related_works":["https://openalex.org/W2355698112","https://openalex.org/W2022984797","https://openalex.org/W2986679525","https://openalex.org/W2797500822","https://openalex.org/W2794458286","https://openalex.org/W4205822456","https://openalex.org/W4299358966","https://openalex.org/W2537367858","https://openalex.org/W2981634480","https://openalex.org/W4288082747"],"abstract_inverted_index":{"Recommender":[0],"models":[1,102],"are":[2,62,108,127],"hard":[3],"to":[4,65],"evaluate,":[5],"particularly":[6],"under":[7],"offline":[8,28,142,189],"setting.":[9],"In":[10],"this":[11],"article,":[12],"we":[13,98,191],"provide":[14],"a":[15,52,54,116,185,193,199],"comprehensive":[16],"and":[17,137,146],"critical":[18],"analysis":[19],"of":[20,78,115,122,174,201],"the":[21,58,69,75,112,120,202],"data":[22,44,87,123,150,157,177],"leakage":[23,31,88,158],"issue":[24],"in":[25,39,178,184,188],"recommender":[26],"system":[27],"evaluation.":[29],"Data":[30],"is":[32],"caused":[33],"by":[34],"not":[35,47,63,109],"observing":[36],"global":[37,49,82],"timeline":[38,194],"evaluating":[40],"recommenders":[41],"e.g.,":[42],"train/test":[43],"split":[45],"does":[46,159],"follow":[48],"timeline.":[50],"As":[51],"result,":[53],"model":[55,204],"learns":[56],"from":[57],"user-item":[59,79],"interactions":[60,80],"that":[61,100,107,156],"expected":[64],"be":[66],"available":[67,110],"at":[68,111],"prediction":[70],"time.":[71],"We":[72,153],"first":[73],"show":[74,99,155],"temporal":[76],"dynamics":[77],"along":[81],"timeline,":[83],"then":[84],"explain":[85],"why":[86],"exists":[89],"for":[90,198],"collaborative":[91],"filtering":[92],"models.":[93],"Through":[94],"carefully":[95],"designed":[96],"experiments,":[97],"all":[101],"indeed":[103],"recommend":[104],"future":[105,176],"items":[106],"time":[113],"point":[114],"test":[117],"instance,":[118],"as":[119],"result":[121],"leakage.":[124],"The":[125],"experiments":[126],"conducted":[128],"with":[129,171],"four":[130,140],"widely":[131],"used":[132],"baseline":[133],"models\u2014BPR,":[134],"NeuMF,":[135],"SASRec,":[136],"LightGCN,":[138],"on":[139],"popular":[141],"datasets\u2014MovieLens-25M,":[143],"Yelp,":[144],"Amazon-music,":[145],"Amazon-electronic,":[147],"adopting":[148],"leave-last-one-out":[149],"split.":[151],"1":[152],"further":[154],"impact":[160],"models\u2019":[161],"recommendation":[162,182,203],"accuracy.":[163],"Their":[164],"relative":[165],"performance":[166],"orders":[167],"thus":[168],"become":[169],"unpredictable":[170],"different":[172],"amount":[173],"leaked":[175],"training.":[179],"To":[180],"evaluate":[181],"systems":[183],"realistic":[186],"manner":[187],"setting,":[190],"propose":[192],"scheme,":[195],"which":[196],"calls":[197],"revisit":[200],"design.":[205]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":30},{"year":2024,"cited_by_count":19},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}