{"id":"https://openalex.org/W4385373885","doi":"https://doi.org/10.1145/3604915.3608839","title":"Widespread Flaws in Offline Evaluation of Recommender Systems","display_name":"Widespread Flaws in Offline Evaluation of Recommender Systems","publication_year":2023,"publication_date":"2023-09-14","ids":{"openalex":"https://openalex.org/W4385373885","doi":"https://doi.org/10.1145/3604915.3608839"},"language":"en","primary_location":{"id":"doi:10.1145/3604915.3608839","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3604915.3608839","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2307.14951","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012061499","display_name":"Bal\u00e1zs Hidasi","orcid":"https://orcid.org/0009-0004-4259-8781"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Bal\u00e1zs Hidasi","raw_affiliation_strings":["Algorithms, Gravity R&D, a Taboola company, Hungary"],"raw_orcid":"https://orcid.org/0009-0004-4259-8781","affiliations":[{"raw_affiliation_string":"Algorithms, Gravity R&D, a Taboola company, Hungary","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5045743963","display_name":"\u00c1d\u00e1m Tibor Czapp","orcid":"https://orcid.org/0000-0001-9576-2080"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"\u00c1d\u00e1m Tibor Czapp","raw_affiliation_strings":["Algorithms, Gravity R&D, a Taboola company, Hungary"],"raw_orcid":"https://orcid.org/0000-0001-9576-2080","affiliations":[{"raw_affiliation_string":"Algorithms, Gravity R&D, a Taboola company, Hungary","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5012061499"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":10.3136,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":{"value":0.98198899,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"848","last_page":"855"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8508542776107788},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8398929238319397},{"id":"https://openalex.org/keywords/copying","display_name":"Copying","score":0.7224354147911072},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4795929491519928},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.4177244305610657},{"id":"https://openalex.org/keywords/proxy","display_name":"Proxy (statistics)","score":0.4121955335140228},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.34443098306655884},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.3034968972206116},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1803278625011444}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8508542776107788},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8398929238319397},{"id":"https://openalex.org/C2779151265","wikidata":"https://www.wikidata.org/wiki/Q1156791","display_name":"Copying","level":2,"score":0.7224354147911072},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4795929491519928},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.4177244305610657},{"id":"https://openalex.org/C2780148112","wikidata":"https://www.wikidata.org/wiki/Q1432581","display_name":"Proxy (statistics)","level":2,"score":0.4121955335140228},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.34443098306655884},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.3034968972206116},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1803278625011444},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3604915.3608839","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3604915.3608839","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 17th ACM Conference on Recommender Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2307.14951","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.14951","pdf_url":"https://arxiv.org/pdf/2307.14951","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2307.14951","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2307.14951","pdf_url":"https://arxiv.org/pdf/2307.14951","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.4300000071525574,"id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G3356421379","display_name":null,"funder_award_id":"2020-1.1.2-PIACI-KFI","funder_id":"https://openalex.org/F4320336675","funder_display_name":"National Research, Development and Innovation Office"}],"funders":[{"id":"https://openalex.org/F4320336675","display_name":"National Research, Development and Innovation Office","ror":null}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385373885.pdf"},"referenced_works_count":46,"referenced_works":["https://openalex.org/W139044672","https://openalex.org/W1921081011","https://openalex.org/W1967787801","https://openalex.org/W2063468305","https://openalex.org/W2097638479","https://openalex.org/W2099419573","https://openalex.org/W2101409192","https://openalex.org/W2144807535","https://openalex.org/W2340502990","https://openalex.org/W2469952266","https://openalex.org/W2511414107","https://openalex.org/W2604662567","https://openalex.org/W2605350416","https://openalex.org/W2626454364","https://openalex.org/W2783272285","https://openalex.org/W2795199972","https://openalex.org/W2798385737","https://openalex.org/W2893359107","https://openalex.org/W2937556626","https://openalex.org/W2945827670","https://openalex.org/W2957191877","https://openalex.org/W2963367478","https://openalex.org/W2984100107","https://openalex.org/W2996931760","https://openalex.org/W3065542300","https://openalex.org/W3081170586","https://openalex.org/W3089147479","https://openalex.org/W3092306916","https://openalex.org/W3100260481","https://openalex.org/W3100278010","https://openalex.org/W3101063193","https://openalex.org/W3102201080","https://openalex.org/W3102619277","https://openalex.org/W3105114834","https://openalex.org/W3185773347","https://openalex.org/W3200556911","https://openalex.org/W3200739262","https://openalex.org/W3201149665","https://openalex.org/W3201178526","https://openalex.org/W3211255867","https://openalex.org/W4224110370","https://openalex.org/W4296604441","https://openalex.org/W4296604546","https://openalex.org/W4296604567","https://openalex.org/W4299286960","https://openalex.org/W4389316291"],"related_works":["https://openalex.org/W4308771405","https://openalex.org/W2355873265","https://openalex.org/W2963669501","https://openalex.org/W3112369086","https://openalex.org/W3080197661","https://openalex.org/W4318471783","https://openalex.org/W2760667490","https://openalex.org/W2991781269","https://openalex.org/W775724729","https://openalex.org/W2137489486"],"abstract_inverted_index":{"Even":[0],"though":[1],"offline":[2,62,109],"evaluation":[3,29,63,76,92,110],"is":[4,59],"just":[5],"an":[6],"imperfect":[7],"proxy":[8],"of":[9,18,28,42,48,54,104,108,111,117],"online":[10,55],"performance":[11],"\u2013":[12,20],"due":[13,86],"to":[14,87],"the":[15,25,35,39,102,106],"interactive":[16],"nature":[17,41],"recommenders":[19,44],"it":[21,58],"will":[22],"probably":[23],"remain":[24],"primary":[26],"way":[27],"in":[30,81],"recommender":[31,82,112],"systems":[32,83],"research":[33,84],"for":[34],"foreseeable":[36],"future,":[37],"since":[38],"proprietary":[40],"production":[43],"prevents":[45],"independent":[46],"validation":[47],"A/B":[49],"test":[50],"setups":[51,64,93],"and":[52,68,121],"verification":[53],"results.":[56],"Therefore,":[57],"imperative":[60],"that":[61],"are":[65,78],"as":[66,69,71],"realistic":[67],"flawless":[70],"they":[72],"can":[73],"be.":[74],"Unfortunately,":[75],"flaws":[77,120],"quite":[79],"common":[80],"nowadays,":[85],"later":[88],"works":[89],"copying":[90],"flawed":[91],"from":[94],"their":[95,99],"predecessors":[96],"without":[97],"questioning":[98],"validity.":[100],"In":[101],"hope":[103],"improving":[105],"quality":[107],"systems,":[113],"we":[114],"discuss":[115],"four":[116],"these":[118],"widespread":[119],"why":[122],"researchers":[123],"should":[124],"avoid":[125],"them.":[126]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":15},{"year":2024,"cited_by_count":7}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}