{"id":"https://openalex.org/W7169518679","doi":"https://doi.org/10.1145/3831686","title":"Reproducibility in Recommender Systems: A Survey","display_name":"Reproducibility in Recommender Systems: A Survey","publication_year":2026,"publication_date":"2026-07-17","ids":{"openalex":"https://openalex.org/W7169518679","doi":"https://doi.org/10.1145/3831686"},"language":"en","primary_location":{"id":"doi:10.1145/3831686","doi":"https://doi.org/10.1145/3831686","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3831686","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-c","cwts-core","jufo-1","norway-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"volume":null,"type":"article","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3831686","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5040472816","display_name":"Alan Said","orcid":"https://orcid.org/0000-0002-2929-0529","observed_orcids":["https://orcid.org/0000-0002-2929-0529"]},"institutions":[{"id":"https://openalex.org/I881427289","display_name":"University of Gothenburg","ror":"https://ror.org/01tm6cn81","country_code":"SE","type":"education","lineage":["https://openalex.org/I881427289"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Alan Said","raw_affiliation_strings":["University of Gothenburg"],"raw_orcid":"https://orcid.org/0000-0002-2929-0529","affiliations":[{"raw_affiliation_string":"University of Gothenburg","institution_ids":["https://openalex.org/I881427289"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029757626","display_name":"Alejandro Bellog\u00edn","orcid":"https://orcid.org/0000-0001-6368-2510","observed_orcids":["https://orcid.org/0000-0001-6368-2510"]},"institutions":[{"id":"https://openalex.org/I63634437","display_name":"Universidad Aut\u00f3noma de Madrid","ror":"https://ror.org/01cby8j38","country_code":"ES","type":"education","lineage":["https://openalex.org/I63634437"]}],"countries":["ES"],"is_corresponding":false,"raw_author_name":"Alejandro Bellogin","raw_affiliation_strings":["Universidad Autonoma de Madrid"],"raw_orcid":"https://orcid.org/0000-0001-6368-2510","affiliations":[{"raw_affiliation_string":"Universidad Autonoma de Madrid","institution_ids":["https://openalex.org/I63634437"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.6254,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.89982833,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"5","issue":"1","first_page":"1","last_page":"23"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.00430000014603138,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10286","display_name":"Information Retrieval and Search Behavior","score":0.00039999998989515007,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"reproducibility","score":0.9879999756813049},{"id":"https://openalex.org/keywords/recommender-systems","display_name":"recommender systems","score":0.9660000205039978},{"id":"https://openalex.org/keywords/replication","display_name":"replication","score":0.49000000953674316},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.9279999732971191},{"id":"https://openalex.org/keywords/research-transparency","display_name":"research transparency","score":0.6340000033378601},{"id":"https://openalex.org/keywords/assessment-practices","display_name":"assessment practices","score":0.5899999737739563},{"id":"https://openalex.org/keywords/research-methodology","display_name":"research methodology","score":0.3440000116825104}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.8704000115394592},{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.7577000260353088},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6402999758720398},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6083999872207642},{"id":"https://openalex.org/C12590798","wikidata":"https://www.wikidata.org/wiki/Q3933199","display_name":"Replication (statistics)","level":2,"score":0.5149999856948853},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.47130000591278076},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.37369999289512634},{"id":"https://openalex.org/C9354725","wikidata":"https://www.wikidata.org/wiki/Q286017","display_name":"Operationalization","level":2,"score":0.36880001425743103},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.3628999888896942},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.3537999987602234},{"id":"https://openalex.org/C2780616401","wikidata":"https://www.wikidata.org/wiki/Q1133673","display_name":"Cornerstone","level":2,"score":0.32030001282691956},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.3167000114917755},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.30640000104904175},{"id":"https://openalex.org/C188087704","wikidata":"https://www.wikidata.org/wiki/Q369577","display_name":"Standardization","level":2,"score":0.2962999939918518},{"id":"https://openalex.org/C63882131","wikidata":"https://www.wikidata.org/wiki/Q17122954","display_name":"Strengths and weaknesses","level":2,"score":0.29319998621940613},{"id":"https://openalex.org/C158154518","wikidata":"https://www.wikidata.org/wiki/Q7310970","display_name":"Relevance (law)","level":2,"score":0.29109999537467957},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.28459998965263367},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2667999863624573},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.2646999955177307}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3831686","doi":"https://doi.org/10.1145/3831686","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3831686","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-c","cwts-core","jufo-1","norway-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2607.26074","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2607.26074","pdf_url":"https://arxiv.org/pdf/2607.26074","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:repositorio.uam.es:10486/794721","doi":null,"is_oa":true,"landing_page_url":"https://hdl.handle.net/10486/794721","pdf_url":null,"source":{"id":"https://openalex.org/S4306400964","display_name":"Biblos-e Archivo (Universidad Aut\u00f3noma de Madrid)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I63634437","host_organization_name":"Universidad Aut\u00f3noma de Madrid","host_organization_lineage":["https://openalex.org/I63634437"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"research article"},{"id":"doi:10.48550/arxiv.2607.26074","doi":"https://doi.org/10.48550/arxiv.2607.26074","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.26074","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3831686","doi":"https://doi.org/10.1145/3831686","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3831686","pdf_url":null,"source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["ccf-c","cwts-core","jufo-1","norway-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1622782223","display_name":null,"funder_award_id":"MCIN/AEI/10","funder_id":"https://openalex.org/F4320335322","funder_display_name":"European Regional Development Fund"},{"id":"https://openalex.org/G3480869486","display_name":null,"funder_award_id":"13039","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G4526231891","display_name":null,"funder_award_id":"MCIN / AEI/10","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G7084143925","display_name":null,"funder_award_id":"AEI/10","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"},{"id":"https://openalex.org/G7970940022","display_name":null,"funder_award_id":"PID2022-139131NB-I00","funder_id":"https://openalex.org/F4320335598","funder_display_name":"Agencia Estatal de Investigaci\u00f3n"}],"funders":[{"id":"https://openalex.org/F4320335322","display_name":"European Regional Development Fund","ror":"https://ror.org/00k4n6c32"},{"id":"https://openalex.org/F4320335598","display_name":"Agencia Estatal de Investigaci\u00f3n","ror":"https://ror.org/003x0zc53"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7169518679.pdf","grobid_xml":"https://content.openalex.org/works/W7169518679.grobid-xml"},"referenced_works_count":79,"referenced_works":["https://openalex.org/W1981575590","https://openalex.org/W1991664944","https://openalex.org/W2054141820","https://openalex.org/W2078488685","https://openalex.org/W2080356852","https://openalex.org/W2101409192","https://openalex.org/W2262817822","https://openalex.org/W2309034743","https://openalex.org/W2605350416","https://openalex.org/W2904474160","https://openalex.org/W2957191877","https://openalex.org/W2963655167","https://openalex.org/W2970236742","https://openalex.org/W2984100107","https://openalex.org/W3028108055","https://openalex.org/W3038719422","https://openalex.org/W3045200674","https://openalex.org/W3081170586","https://openalex.org/W3088444111","https://openalex.org/W3127475352","https://openalex.org/W3134063549","https://openalex.org/W3183638647","https://openalex.org/W3185773347","https://openalex.org/W3199892213","https://openalex.org/W3208227120","https://openalex.org/W3211255867","https://openalex.org/W4212961739","https://openalex.org/W4225321042","https://openalex.org/W4249267926","https://openalex.org/W4292092796","https://openalex.org/W4296591867","https://openalex.org/W4296604468","https://openalex.org/W4296604567","https://openalex.org/W4385373976","https://openalex.org/W4386728770","https://openalex.org/W4386728819","https://openalex.org/W4386728823","https://openalex.org/W4386728847","https://openalex.org/W4386729058","https://openalex.org/W4386729258","https://openalex.org/W4387848563","https://openalex.org/W4387999295","https://openalex.org/W4392367398","https://openalex.org/W4403219726","https://openalex.org/W4403219808","https://openalex.org/W4403220207","https://openalex.org/W4403220390","https://openalex.org/W4403220425","https://openalex.org/W4403220476","https://openalex.org/W4403220643","https://openalex.org/W4403220697","https://openalex.org/W4403220774","https://openalex.org/W4403220861","https://openalex.org/W4403220878","https://openalex.org/W4403220991","https://openalex.org/W4403221560","https://openalex.org/W4403221577","https://openalex.org/W4409183120","https://openalex.org/W4414034759","https://openalex.org/W4414034768","https://openalex.org/W4414034785","https://openalex.org/W4414034793","https://openalex.org/W4414034802","https://openalex.org/W4414034808","https://openalex.org/W4414034814","https://openalex.org/W4414034838","https://openalex.org/W4414034841","https://openalex.org/W4414034851","https://openalex.org/W4414034868","https://openalex.org/W4414034874","https://openalex.org/W4414034911","https://openalex.org/W4414034916","https://openalex.org/W4414034945","https://openalex.org/W4414034956","https://openalex.org/W4414034969","https://openalex.org/W4414034979","https://openalex.org/W4414035029","https://openalex.org/W4414035031","https://openalex.org/W4414035066"],"related_works":["https://openalex.org/W1991664944","https://openalex.org/W1969644815","https://openalex.org/W4414034759","https://openalex.org/W7100859899","https://openalex.org/W7097565569","https://openalex.org/W2309034743","https://openalex.org/W4376873982","https://openalex.org/W7213747525","https://openalex.org/W4229000037","https://openalex.org/W4385373976"],"abstract_inverted_index":{"Reproducibility":[0,31],"has":[1,98,162,168],"become":[2],"a":[3,29,45,104,121,127,176],"cornerstone":[4],"of":[5,19,48,78,130,182],"credible":[6],"recommender":[7],"systems":[8],"research,":[9],"driven":[10],"by":[11,62],"growing":[12],"concerns":[13],"about":[14],"the":[15,24,49,76,87,96,179],"reliability":[16],"and":[17,39,64,72,108,114,133,165,184],"generalizability":[18],"experimental":[20],"results.":[21],"In":[22],"response,":[23],"ACM":[25],"RecSys":[26],"conference":[27],"introduced":[28],"dedicated":[30],"Track":[32],"in":[33,68,84,100,138],"2020":[34,52],"to":[35,53,110],"encourage":[36],"rigorous,":[37],"transparent,":[38],"repeatable":[40],"research.":[41],"This":[42],"article":[43],"presents":[44],"structured":[46],"analysis":[47],"track":[50,97],"from":[51,103],"2025,":[54],"covering":[55],"51":[56],"accepted":[57],"papers.":[58],"We":[59],"classify":[60],"contributions":[61],"type":[63],"analyze":[65],"common":[66],"patterns":[67],"datasets,":[69,131],"algorithms,":[70,132],"frameworks,":[71],"evaluation":[73,134,157],"practices,":[74],"with":[75,150],"goal":[77],"understanding":[79],"how":[80],"reproducibility":[81,118,137,160,183],"is":[82],"operationalized":[83],"practice":[85,139],"within":[86],"community.":[88],"Our":[89],"findings":[90],"reveal":[91],"three":[92],"main":[93],"trends.":[94],"First,":[95],"expanded":[99],"scope,":[101],"evolving":[102],"focus":[105],"on":[106,126,172],"reproduction":[107],"replication":[109],"include":[111],"benchmarking,":[112],"resources,":[113],"methodological":[115,123,173],"contributions.":[116],"Second,":[117],"papers":[119],"exhibit":[120],"consistent":[122],"profile,":[124],"relying":[125],"limited":[128,170],"set":[129],"protocols.":[135],"Third,":[136],"often":[140],"involves":[141],"extending":[142],"prior":[143],"experiments":[144],"rather":[145],"than":[146],"strictly":[147],"replicating":[148],"them,":[149],"studies":[151],"frequently":[152],"introducing":[153],"additional":[154],"models":[155],"or":[156],"criteria.":[158],"Overall,":[159],"work":[161],"improved":[163],"transparency":[164],"documentation,":[166],"but":[167],"had":[169],"impact":[171],"diversity,":[174],"highlighting":[175],"gap":[177],"between":[178],"conceptual":[180],"definition":[181],"its":[185],"implementation.":[186]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2026-07-18T00:00:00"}
