{"id":"https://openalex.org/W4376873982","doi":"https://doi.org/10.1145/3596519","title":"A Systematic Study on Reproducibility of Reinforcement Learning in Recommendation Systems","display_name":"A Systematic Study on Reproducibility of Reinforcement Learning in Recommendation Systems","publication_year":2023,"publication_date":"2023-05-17","ids":{"openalex":"https://openalex.org/W4376873982","doi":"https://doi.org/10.1145/3596519"},"language":"en","primary_location":{"id":"doi:10.1145/3596519","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3596519","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5002642243","display_name":"Emanuele Cavenaghi","orcid":"https://orcid.org/0000-0002-0235-0421"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Emanuele Cavenaghi","raw_affiliation_strings":["Free University of Bozen-Bolzano, Italy"],"raw_orcid":"https://orcid.org/0000-0002-0235-0421","affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Italy","institution_ids":["https://openalex.org/I171543936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016449483","display_name":"Gabriele Sottocornola","orcid":"https://orcid.org/0000-0001-9983-2330"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Gabriele Sottocornola","raw_affiliation_strings":["Free University of Bozen-Bolzano, Italy"],"raw_orcid":"https://orcid.org/0000-0001-9983-2330","affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Italy","institution_ids":["https://openalex.org/I171543936"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020281960","display_name":"Fabio Stella","orcid":"https://orcid.org/0000-0002-1394-0507"},"institutions":[{"id":"https://openalex.org/I66752286","display_name":"University of Milano-Bicocca","ror":"https://ror.org/01ynf4891","country_code":"IT","type":"education","lineage":["https://openalex.org/I66752286"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Fabio Stella","raw_affiliation_strings":["University of Milano-Bicocca, Italy"],"raw_orcid":"https://orcid.org/0000-0002-1394-0507","affiliations":[{"raw_affiliation_string":"University of Milano-Bicocca, Italy","institution_ids":["https://openalex.org/I66752286"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5039093142","display_name":"Markus Zanker","orcid":"https://orcid.org/0000-0002-4805-5516"},"institutions":[{"id":"https://openalex.org/I171543936","display_name":"Free University of Bozen-Bolzano","ror":"https://ror.org/012ajp527","country_code":"IT","type":"education","lineage":["https://openalex.org/I171543936"]},{"id":"https://openalex.org/I4210166741","display_name":"University of Klagenfurt","ror":"https://ror.org/05q9m0937","country_code":"AT","type":"education","lineage":["https://openalex.org/I4210166741"]}],"countries":["AT","IT"],"is_corresponding":false,"raw_author_name":"Markus Zanker","raw_affiliation_strings":["Free University of Bozen-Bolzano, Italy and University of Klagenfurt, Austria"],"raw_orcid":"https://orcid.org/0000-0002-4805-5516","affiliations":[{"raw_affiliation_string":"Free University of Bozen-Bolzano, Italy and University of Klagenfurt, Austria","institution_ids":["https://openalex.org/I171543936","https://openalex.org/I4210166741"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.6522,"has_fulltext":false,"cited_by_count":7,"citation_normalized_percentile":{"value":0.84452466,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"1","issue":"3","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.8260012865066528},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7755793929100037},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.6611940860748291},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.6043149828910828},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5729661583900452},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.5349017381668091},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.4895082712173462},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4505324363708496},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4456016421318054},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3535583019256592},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.30034804344177246},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09275209903717041},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07875704765319824},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.06099238991737366}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.8260012865066528},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7755793929100037},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6611940860748291},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.6043149828910828},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5729661583900452},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.5349017381668091},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.4895082712173462},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4505324363708496},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4456016421318054},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3535583019256592},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.30034804344177246},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09275209903717041},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07875704765319824},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.06099238991737366},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3596519","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3596519","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Gender equality","score":0.4099999964237213,"id":"https://metadata.un.org/sdg/5"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320325400","display_name":"Libera Universit\u00e0 di Bolzano","ror":"https://ror.org/012ajp527"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":79,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W1486317198","https://openalex.org/W1552276231","https://openalex.org/W1583837637","https://openalex.org/W1998498767","https://openalex.org/W2028627965","https://openalex.org/W2076337359","https://openalex.org/W2155106456","https://openalex.org/W2169004268","https://openalex.org/W2531563875","https://openalex.org/W2605350416","https://openalex.org/W2759912774","https://openalex.org/W2765982206","https://openalex.org/W2766397879","https://openalex.org/W2787225861","https://openalex.org/W2787933113","https://openalex.org/W2788295351","https://openalex.org/W2788572835","https://openalex.org/W2797234205","https://openalex.org/W2799117832","https://openalex.org/W2799544270","https://openalex.org/W2809162153","https://openalex.org/W2897405591","https://openalex.org/W2902572901","https://openalex.org/W2905101140","https://openalex.org/W2911104099","https://openalex.org/W2936427778","https://openalex.org/W2941385591","https://openalex.org/W2946327583","https://openalex.org/W2950275995","https://openalex.org/W2963561234","https://openalex.org/W2963619374","https://openalex.org/W2964112275","https://openalex.org/W2965512832","https://openalex.org/W2969920248","https://openalex.org/W2981211936","https://openalex.org/W2982120135","https://openalex.org/W2996736458","https://openalex.org/W2996959725","https://openalex.org/W2997556638","https://openalex.org/W2997561372","https://openalex.org/W2997914278","https://openalex.org/W2999560001","https://openalex.org/W3003341035","https://openalex.org/W3007830800","https://openalex.org/W3011809564","https://openalex.org/W3012881846","https://openalex.org/W3013688454","https://openalex.org/W3017289192","https://openalex.org/W3020831971","https://openalex.org/W3032675404","https://openalex.org/W3034791416","https://openalex.org/W3034833075","https://openalex.org/W3034853385","https://openalex.org/W3035084859","https://openalex.org/W3035166059","https://openalex.org/W3035170973","https://openalex.org/W3040094401","https://openalex.org/W3047443212","https://openalex.org/W3049342604","https://openalex.org/W3088777230","https://openalex.org/W3090790792","https://openalex.org/W3102778384","https://openalex.org/W3102899483","https://openalex.org/W3106445281","https://openalex.org/W3116249021","https://openalex.org/W3123348991","https://openalex.org/W3123956618","https://openalex.org/W3146823280","https://openalex.org/W3173331009","https://openalex.org/W3197433610","https://openalex.org/W3201802316","https://openalex.org/W3208231292","https://openalex.org/W4206275166","https://openalex.org/W4212774754","https://openalex.org/W4212961739","https://openalex.org/W4287389313","https://openalex.org/W4318159260","https://openalex.org/W6669402789"],"related_works":["https://openalex.org/W3159063414","https://openalex.org/W4295309597","https://openalex.org/W4210794429","https://openalex.org/W4309113015","https://openalex.org/W4223456145","https://openalex.org/W3199608561","https://openalex.org/W4319083788","https://openalex.org/W4283460463","https://openalex.org/W4360764815","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Reproducibility":[0],"is":[1,134],"a":[2,66,76,137],"main":[3],"principle":[4],"in":[5,27,61,122],"science":[6],"and":[7,71,102],"fundamental":[8],"to":[9,57,80,114,156],"ensure":[10],"scientific":[11],"progress.":[12],"However,":[13],"many":[14,123],"recent":[15],"works":[16],"point":[17],"out":[18],"that":[19,86],"there":[20],"are":[21],"widespread":[22],"deficiencies":[23],"for":[24],"this":[25,62],"aspect":[26],"the":[28,32,43,49,59,82,108,115,127,143,151],"AI":[29,125],"field,":[30],"making":[31],"reproducibility":[33,46,138],"of":[34,45,51,68,78,142],"results":[35],"impractical":[36],"or":[37],"even":[38],"impossible.":[39],"We":[40,64,118],"therefore":[41],"studied":[42],"state":[44],"support":[47],"on":[48],"topic":[50],"Reinforcement":[52,128],"Learning":[53,129],"&":[54,130],"Recommender":[55,131],"Systems":[56,132],"analyse":[58],"situation":[60],"context.":[63],"collected":[65],"total":[67],"60":[69],"papers":[70,145],"analysed":[72],"them":[73,113],"by":[74],"defining":[75],"set":[77],"variables":[79],"inspect":[81],"most":[83],"important":[84],"aspects":[85],"enable":[87],"reproducibility,":[88],"such":[89],"as":[90,140],"dataset,":[91],"pre-processing":[92],"code,":[93],"hardware":[94],"specifications,":[95],"software":[96],"dependencies,":[97],"algorithm":[98,100],"implementation,":[99],"hyperparameters,":[101],"experiment":[103],"code.":[104],"Furthermore,":[105],"we":[106],"used":[107],"ACM":[109,152],"Badges":[110,153],"definitions":[111,154],"assigning":[112],"selected":[116,144],"papers.":[117],"discovered":[119],"that,":[120],"like":[121],"other":[124],"domains,":[126],"field":[133],"grappling":[135],"with":[136],"crisis,":[139],"none":[141],"were":[146],"reproducible":[147],"when":[148],"strictly":[149],"applying":[150],"according":[155],"our":[157],"analysis.":[158]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":2}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}