{"id":"https://openalex.org/W4283587264","doi":"https://doi.org/10.1145/3545796","title":"A Revisiting Study of Appropriate Offline Evaluation for Top- <i>N</i> Recommendation Algorithms","display_name":"A Revisiting Study of Appropriate Offline Evaluation for Top- <i>N</i> Recommendation Algorithms","publication_year":2022,"publication_date":"2022-06-27","ids":{"openalex":"https://openalex.org/W4283587264","doi":"https://doi.org/10.1145/3545796"},"language":"en","primary_location":{"id":"doi:10.1145/3545796","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3545796","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5037145565","display_name":"Wayne Xin Zhao","orcid":"https://orcid.org/0000-0002-8333-6196"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Wayne Xin Zhao","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8333-6196","affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052362113","display_name":"Zihan Lin","orcid":"https://orcid.org/0000-0002-6877-4470"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihan Lin","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-6877-4470","affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025638481","display_name":"Zhichao Feng","orcid":"https://orcid.org/0000-0001-6913-9260"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhichao Feng","raw_affiliation_strings":["Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6913-9260","affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100399541","display_name":"Pengfei Wang","orcid":"https://orcid.org/0000-0001-8658-7102"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pengfei Wang","raw_affiliation_strings":["Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8658-7102","affiliations":[{"raw_affiliation_string":"Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025631695","display_name":"Ji-Rong Wen","orcid":"https://orcid.org/0000-0002-9777-9676"},"institutions":[{"id":"https://openalex.org/I78988378","display_name":"Renmin University of China","ror":"https://ror.org/041pakw92","country_code":"CN","type":"education","lineage":["https://openalex.org/I78988378"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji-Rong Wen","raw_affiliation_strings":["Renmin University of China, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9777-9676","affiliations":[{"raw_affiliation_string":"Renmin University of China, Beijing, China","institution_ids":["https://openalex.org/I78988378"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5037145565"],"corresponding_institution_ids":["https://openalex.org/I78988378"],"apc_list":null,"apc_paid":null,"fwci":7.9645,"has_fulltext":false,"cited_by_count":27,"citation_normalized_percentile":{"value":0.97466563,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"41","issue":"2","first_page":"1","last_page":"41"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9783999919891357,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8793141841888428},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8148316740989685},{"id":"https://openalex.org/keywords/ranking","display_name":"Ranking (information retrieval)","score":0.7003859281539917},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5943638682365417},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5891640782356262},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.5662457346916199},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5506611466407776},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.5255112648010254},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5186806917190552},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.48025229573249817},{"id":"https://openalex.org/keywords/learning-to-rank","display_name":"Learning to rank","score":0.43422698974609375},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.4298170804977417},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3768808841705322},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06391608715057373}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8793141841888428},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8148316740989685},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.7003859281539917},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5943638682365417},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5891640782356262},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.5662457346916199},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5506611466407776},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.5255112648010254},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5186806917190552},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.48025229573249817},{"id":"https://openalex.org/C86037889","wikidata":"https://www.wikidata.org/wiki/Q4330127","display_name":"Learning to rank","level":3,"score":0.43422698974609375},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.4298170804977417},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3768808841705322},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06391608715057373},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3545796","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3545796","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3710898451","display_name":null,"funder_award_id":"61872369 and 61802029","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":144,"referenced_works":["https://openalex.org/W79478496","https://openalex.org/W178169250","https://openalex.org/W1486317198","https://openalex.org/W1886704267","https://openalex.org/W1959619913","https://openalex.org/W1965355809","https://openalex.org/W1976618413","https://openalex.org/W1981575590","https://openalex.org/W1994389483","https://openalex.org/W2016043834","https://openalex.org/W2027731328","https://openalex.org/W2045745608","https://openalex.org/W2049670925","https://openalex.org/W2049750575","https://openalex.org/W2054141820","https://openalex.org/W2055945388","https://openalex.org/W2077927809","https://openalex.org/W2078488685","https://openalex.org/W2097998348","https://openalex.org/W2101409192","https://openalex.org/W2106666261","https://openalex.org/W2108630796","https://openalex.org/W2108920354","https://openalex.org/W2114079787","https://openalex.org/W2117643431","https://openalex.org/W2124187902","https://openalex.org/W2134040744","https://openalex.org/W2142144955","https://openalex.org/W2189280035","https://openalex.org/W2219888463","https://openalex.org/W2226205360","https://openalex.org/W2253995343","https://openalex.org/W2295739661","https://openalex.org/W2340502990","https://openalex.org/W2509235963","https://openalex.org/W2511414107","https://openalex.org/W2604272474","https://openalex.org/W2605350416","https://openalex.org/W2608239929","https://openalex.org/W2613725451","https://openalex.org/W2626454364","https://openalex.org/W2725606191","https://openalex.org/W2739273093","https://openalex.org/W2740305962","https://openalex.org/W2740429388","https://openalex.org/W2740920897","https://openalex.org/W2748276617","https://openalex.org/W2748374161","https://openalex.org/W2767696711","https://openalex.org/W2767724106","https://openalex.org/W2774321216","https://openalex.org/W2783666221","https://openalex.org/W2783792211","https://openalex.org/W2784068709","https://openalex.org/W2787900111","https://openalex.org/W2788456731","https://openalex.org/W2788663865","https://openalex.org/W2798868970","https://openalex.org/W2798972759","https://openalex.org/W2802187397","https://openalex.org/W2804022707","https://openalex.org/W2807848202","https://openalex.org/W2808751420","https://openalex.org/W2883308936","https://openalex.org/W2884134047","https://openalex.org/W2892888989","https://openalex.org/W2892923641","https://openalex.org/W2896367309","https://openalex.org/W2896691185","https://openalex.org/W2900229157","https://openalex.org/W2905305843","https://openalex.org/W2907754626","https://openalex.org/W2907827821","https://openalex.org/W2912351665","https://openalex.org/W2912570479","https://openalex.org/W2913525628","https://openalex.org/W2914904117","https://openalex.org/W2944441143","https://openalex.org/W2945623882","https://openalex.org/W2945827670","https://openalex.org/W2947656526","https://openalex.org/W2954808392","https://openalex.org/W2957191877","https://openalex.org/W2962992837","https://openalex.org/W2963085847","https://openalex.org/W2963902947","https://openalex.org/W2964965220","https://openalex.org/W2965727822","https://openalex.org/W2972520532","https://openalex.org/W2972675851","https://openalex.org/W2972970099","https://openalex.org/W2974976686","https://openalex.org/W2979057167","https://openalex.org/W2987577970","https://openalex.org/W2996863522","https://openalex.org/W2997823717","https://openalex.org/W2999649805","https://openalex.org/W3009464535","https://openalex.org/W3011685549","https://openalex.org/W3011809564","https://openalex.org/W3012781334","https://openalex.org/W3032861513","https://openalex.org/W3034216953","https://openalex.org/W3034718811","https://openalex.org/W3035589558","https://openalex.org/W3043433718","https://openalex.org/W3045200674","https://openalex.org/W3080566854","https://openalex.org/W3080781457","https://openalex.org/W3080884086","https://openalex.org/W3081170586","https://openalex.org/W3088043608","https://openalex.org/W3088444111","https://openalex.org/W3088694469","https://openalex.org/W3088777230","https://openalex.org/W3089147479","https://openalex.org/W3092103025","https://openalex.org/W3093922720","https://openalex.org/W3094127838","https://openalex.org/W3097991661","https://openalex.org/W3099420497","https://openalex.org/W3100278010","https://openalex.org/W3101830194","https://openalex.org/W3102619277","https://openalex.org/W3102755086","https://openalex.org/W3105114834","https://openalex.org/W3106181667","https://openalex.org/W3106445281","https://openalex.org/W3106450873","https://openalex.org/W3122507327","https://openalex.org/W3125645198","https://openalex.org/W3134041167","https://openalex.org/W3134063549","https://openalex.org/W3154113024","https://openalex.org/W3154190849","https://openalex.org/W3208227120","https://openalex.org/W4212843742","https://openalex.org/W4231054948","https://openalex.org/W4251597219","https://openalex.org/W4299286960","https://openalex.org/W4300879629","https://openalex.org/W4310228395","https://openalex.org/W6674385629","https://openalex.org/W6692935382"],"related_works":["https://openalex.org/W3127142483","https://openalex.org/W4385565564","https://openalex.org/W2138488530","https://openalex.org/W2898073868","https://openalex.org/W2971071571","https://openalex.org/W2798835721","https://openalex.org/W2387658907","https://openalex.org/W2922169395","https://openalex.org/W2385796165","https://openalex.org/W25098770"],"abstract_inverted_index":{"In":[0,99,170],"recommender":[1,72,148],"systems,":[2],"top-":[3,27,78],"N":[4,28,79],"recommendation":[5,38,47,80,84,114],"is":[6,42],"an":[7,111],"important":[8,43,65,185],"task":[9],"with":[10,94],"implicit":[11],"feedback":[12],"data.":[13],"Although":[14],"the":[15,24,102,122,131,143,167],"recent":[16,173],"success":[17],"of":[18,37,126,166],"deep":[19],"learning":[20],"largely":[21],"pushes":[22],"forward":[23],"research":[25],"on":[26,34,63,110,130,142],"recommendation,":[29],"there":[30],"are":[31,88,107,183],"increasing":[32],"concerns":[33],"appropriate":[35],"evaluation":[36,174],"algorithms.":[39,169],"It":[40],"therefore":[41],"to":[44,158,172],"study":[45,62],"how":[46],"algorithms":[48,81,96],"can":[49,156],"be":[50],"reliably":[51],"evaluated":[52],"and":[53,82,91,97,124,134],"thoroughly":[54],"verified.":[55],"This":[56],"work":[57,106],"presents":[58],"a":[59],"large-scale,":[60],"systematic":[61],"six":[64],"factors":[66],"from":[67],"three":[68],"aspects":[69],"for":[70,146,186],"evaluating":[71,147],"systems.":[73,149],"We":[74],"carefully":[75,89],"select":[76],"12":[77],"eight":[83],"datasets.":[85,98],"Our":[86,150],"experiments":[87,103,133],"designed":[90],"extensively":[92],"conducted":[93],"these":[95],"particular,":[100],"all":[101],"in":[104,163],"our":[105,127],"implemented":[108],"based":[109],"open":[112],"sourced":[113],"library,":[115],"Recbole":[116],"[":[117],"139":[118],"],":[119],"which":[120],"ensures":[121],"reproducibility":[123],"reliability":[125],"results.":[128],"Based":[129],"large-scale":[132],"detailed":[135],"analysis,":[136],"we":[137,176],"derive":[138],"several":[139,179],"key":[140],"findings":[141,151],"experimental":[144],"settings":[145,155,181],"show":[152],"that":[153,182],"some":[154],"lead":[157],"substantial":[159],"or":[160],"significant":[161],"differences":[162],"performance":[164,187],"ranking":[165],"compared":[168],"response":[171],"concerns,":[175],"also":[177],"provide":[178],"suggested":[180],"specially":[184],"comparison.":[188]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}