{"id":"https://openalex.org/W2987577970","doi":"https://doi.org/10.1145/3434185","title":"A Troubling Analysis of Reproducibility and Progress in Recommender Systems Research","display_name":"A Troubling Analysis of Reproducibility and Progress in Recommender Systems Research","publication_year":2021,"publication_date":"2021-01-06","ids":{"openalex":"https://openalex.org/W2987577970","doi":"https://doi.org/10.1145/3434185","mag":"2987577970"},"language":"en","primary_location":{"id":"doi:10.1145/3434185","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3434185","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1911.07698","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Maurizio Ferrari Dacrema","orcid":null},"institutions":[{"id":"https://openalex.org/I93860229","display_name":"Politecnico di Milano","ror":"https://ror.org/01nffqt88","country_code":"IT","type":"education","lineage":["https://openalex.org/I93860229"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Maurizio Ferrari Dacrema","raw_affiliation_strings":["Politecnico di Milano, Milano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Politecnico di Milano, Milano, Italy","institution_ids":["https://openalex.org/I93860229"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Simone Boglio","orcid":null},"institutions":[{"id":"https://openalex.org/I93860229","display_name":"Politecnico di Milano","ror":"https://ror.org/01nffqt88","country_code":"IT","type":"education","lineage":["https://openalex.org/I93860229"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Simone Boglio","raw_affiliation_strings":["Politecnico di Milano, Milano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Politecnico di Milano, Milano, Italy","institution_ids":["https://openalex.org/I93860229"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Paolo Cremonesi","orcid":null},"institutions":[{"id":"https://openalex.org/I93860229","display_name":"Politecnico di Milano","ror":"https://ror.org/01nffqt88","country_code":"IT","type":"education","lineage":["https://openalex.org/I93860229"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Paolo Cremonesi","raw_affiliation_strings":["Politecnico di Milano, Milano, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Politecnico di Milano, Milano, Italy","institution_ids":["https://openalex.org/I93860229"]}]},{"author_position":"last","author":{"id":null,"display_name":"Dietmar Jannach","orcid":null},"institutions":[{"id":"https://openalex.org/I4210166741","display_name":"University of Klagenfurt","ror":"https://ror.org/05q9m0937","country_code":"AT","type":"education","lineage":["https://openalex.org/I4210166741"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dietmar Jannach","raw_affiliation_strings":["University of Klagenfurt, Klagenfurt, Austria"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Klagenfurt, Klagenfurt, Austria","institution_ids":["https://openalex.org/I4210166741"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":38.7997,"has_fulltext":false,"cited_by_count":187,"citation_normalized_percentile":{"value":0.99856101,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"39","issue":"2","first_page":"1","last_page":"49"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.840499997138977,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.840499997138977,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14413","display_name":"Advanced Technologies in Various Fields","score":0.012799999676644802,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.01140000019222498,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7702999711036682},{"id":"https://openalex.org/keywords/collaborative-filtering","display_name":"Collaborative filtering","score":0.6974999904632568},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.670199990272522},{"id":"https://openalex.org/keywords/heuristic","display_name":"Heuristic","score":0.603600025177002},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.5681999921798706},{"id":"https://openalex.org/keywords/simple","display_name":"Simple (philosophy)","score":0.5027999877929688},{"id":"https://openalex.org/keywords/matrix-decomposition","display_name":"Matrix decomposition","score":0.36390000581741333}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8707000017166138},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7702999711036682},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.6974999904632568},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.670199990272522},{"id":"https://openalex.org/C173801870","wikidata":"https://www.wikidata.org/wiki/Q201413","display_name":"Heuristic","level":2,"score":0.603600025177002},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5681999921798706},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5320000052452087},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.5027999877929688},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47099998593330383},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42289999127388},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3686000108718872},{"id":"https://openalex.org/C42355184","wikidata":"https://www.wikidata.org/wiki/Q1361088","display_name":"Matrix decomposition","level":3,"score":0.36390000581741333},{"id":"https://openalex.org/C183003079","wikidata":"https://www.wikidata.org/wiki/Q1000371","display_name":"Personalization","level":2,"score":0.36169999837875366},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.3305000066757202},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.29510000348091125},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2904999852180481},{"id":"https://openalex.org/C58489278","wikidata":"https://www.wikidata.org/wiki/Q1172284","display_name":"Data set","level":2,"score":0.2662000060081482},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C2776372474","wikidata":"https://www.wikidata.org/wiki/Q508291","display_name":"Simplicity","level":2,"score":0.2563999891281128},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2506999969482422}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3434185","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3434185","pdf_url":null,"source":{"id":"https://openalex.org/S4394735545","display_name":"ACM Transactions on Information Systems","issn_l":"1046-8188","issn":["1046-8188","1558-2868"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Information Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1911.07698","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1911.07698","pdf_url":"https://arxiv.org/pdf/1911.07698","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:re.public.polimi.it:11311/1164333","is_oa":true,"landing_page_url":"https://hdl.handle.net/11311/1164333","pdf_url":null,"source":{"id":"https://openalex.org/S4306400312","display_name":"Virtual Community of Pathological Anatomy (University of Castilla La Mancha)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I79189158","host_organization_name":"University of Castilla-La Mancha","host_organization_lineage":["https://openalex.org/I79189158"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1911.07698","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1911.07698","pdf_url":"https://arxiv.org/pdf/1911.07698","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":62,"referenced_works":["https://openalex.org/W1587374567","https://openalex.org/W1987431925","https://openalex.org/W1987869189","https://openalex.org/W1992000819","https://openalex.org/W2006996936","https://openalex.org/W2012903477","https://openalex.org/W2018371603","https://openalex.org/W2042281163","https://openalex.org/W2049670925","https://openalex.org/W2055945388","https://openalex.org/W2059975159","https://openalex.org/W2101409192","https://openalex.org/W2114079787","https://openalex.org/W2126695993","https://openalex.org/W2139210474","https://openalex.org/W2150886314","https://openalex.org/W2155106456","https://openalex.org/W2157881433","https://openalex.org/W2159094788","https://openalex.org/W2163162311","https://openalex.org/W2172296140","https://openalex.org/W2196920274","https://openalex.org/W2207136466","https://openalex.org/W2253995343","https://openalex.org/W2283438976","https://openalex.org/W2511414107","https://openalex.org/W2512030163","https://openalex.org/W2515144511","https://openalex.org/W2605350416","https://openalex.org/W2618756248","https://openalex.org/W2725606191","https://openalex.org/W2740920897","https://openalex.org/W2741249238","https://openalex.org/W2745534091","https://openalex.org/W2783565819","https://openalex.org/W2786995169","https://openalex.org/W2795199972","https://openalex.org/W2798749602","https://openalex.org/W2798972759","https://openalex.org/W2808482351","https://openalex.org/W2808490894","https://openalex.org/W2893543100","https://openalex.org/W2893775232","https://openalex.org/W2893785281","https://openalex.org/W2895251367","https://openalex.org/W2910577570","https://openalex.org/W2912745432","https://openalex.org/W2957191877","https://openalex.org/W2962960341","https://openalex.org/W2963085847","https://openalex.org/W2963636167","https://openalex.org/W2964169350","https://openalex.org/W2972941122","https://openalex.org/W3011685549","https://openalex.org/W3041344680","https://openalex.org/W3044177754","https://openalex.org/W3044549925","https://openalex.org/W3088777230","https://openalex.org/W3136109765","https://openalex.org/W4237712395","https://openalex.org/W4288280763","https://openalex.org/W4300175872"],"related_works":["https://openalex.org/W2348159088","https://openalex.org/W2402445420","https://openalex.org/W1570823782","https://openalex.org/W2058862692","https://openalex.org/W1479993970","https://openalex.org/W937687500","https://openalex.org/W4289865866","https://openalex.org/W1773619406","https://openalex.org/W2323023697","https://openalex.org/W2294569909"],"abstract_inverted_index":{"The":[0,116],"design":[1],"of":[2,14,19,44,56,71,88,100,112,119,122,138,161,214],"algorithms":[3],"that":[4,136,199],"generate":[5],"personalized":[6],"ranked":[7],"item":[8],"lists":[9],"is":[10,51],"a":[11,85,109,211],"central":[12],"topic":[13],"research":[15,61,192],"in":[16,27,39,59,97,190],"the":[17,23,40,49,67,72,80,89,98,120,139,154,162,196,203,208],"field":[18,209],"recommender":[20],"systems.":[21],"In":[22,183],"past":[24],"few":[25],"years,":[26],"particular,":[28],"approaches":[29,103,143],"based":[30,104,152],"on":[31,105,153,202],"deep":[32],"learning":[33],"(neural)":[34],"techniques":[35],"have":[36,93,205],"become":[37],"dominant":[38],"literature.":[41],"For":[42],"all":[43],"them,":[45],"substantial":[46],"progress":[47],"over":[48],"state-of-the-art":[50],"claimed.":[52],"However,":[53],"indications":[54],"exist":[55],"certain":[57,212],"problems":[58],"today\u2019s":[60,191],"practice,":[62,193],"e.g.,":[63,151,176],"with":[64],"respect":[65],"to":[66,210],"choice":[68],"and":[69,134],"optimization":[70],"baselines":[73],"used":[74],"for":[75],"comparison,":[76],"raising":[77],"questions":[78],"about":[79],"published":[81,127,201],"claims.":[82],"To":[83],"obtain":[84],"better":[86,170],"understanding":[87],"actual":[90],"progress,":[91],"we":[92,186],"compared":[94],"recent":[95,124],"results":[96],"area":[99],"neural":[101,142,165],"recommendation":[102],"collaborative":[106],"filtering":[107],"against":[108],"consistent":[110],"set":[111],"existing":[113,173],"simple":[114,149],"baselines.":[115],"worrying":[117],"outcome":[118],"analysis":[121],"these":[123],"works\u2014all":[125],"were":[126],"at":[128],"prestigious":[129],"scientific":[130],"conferences":[131],"between":[132],"2015":[133],"2018\u2014is":[135],"11":[137],"12":[140],"reproducible":[141],"can":[144],"be":[145],"outperformed":[146],"by":[147],"conceptually":[148],"methods,":[150],"nearest-neighbor":[155],"heuristic":[156],"or":[157,180],"linear":[158,181],"models.":[159,182],"None":[160],"computationally":[163],"complex":[164],"methods":[166],"was":[167],"actually":[168],"consistently":[169],"than":[171],"already":[172],"learning-based":[174],"techniques,":[175],"using":[177],"matrix":[178],"factorization":[179],"our":[184],"analysis,":[185],"discuss":[187],"common":[188],"issues":[189],"which,":[194],"despite":[195],"many":[197],"papers":[198],"are":[200],"topic,":[204],"apparently":[206],"led":[207],"level":[213],"stagnation.":[215],"1":[216]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":34},{"year":2024,"cited_by_count":49},{"year":2023,"cited_by_count":38},{"year":2022,"cited_by_count":34},{"year":2021,"cited_by_count":19},{"year":2020,"cited_by_count":2},{"year":2012,"cited_by_count":3}],"updated_date":"2026-07-11T18:08:03.149640","created_date":"2019-11-22T00:00:00"}