{"id":"https://openalex.org/W4414034759","doi":"https://doi.org/10.1145/3705328.3748157","title":"\u201cWe Share Our Code Online\u201d: Why This Is Not Enough to Ensure Reproducibility and Progress in Recommender Systems Research","display_name":"\u201cWe Share Our Code Online\u201d: Why This Is Not Enough to Ensure Reproducibility and Progress in Recommender Systems Research","publication_year":2025,"publication_date":"2025-09-06","ids":{"openalex":"https://openalex.org/W4414034759","doi":"https://doi.org/10.1145/3705328.3748157"},"language":"en","primary_location":{"id":"doi:10.1145/3705328.3748157","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748157","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3705328.3748157","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5071874921","display_name":"Faisal Shehzad","orcid":"https://orcid.org/0000-0001-6239-8198"},"institutions":[{"id":"https://openalex.org/I4210166741","display_name":"University of Klagenfurt","ror":"https://ror.org/05q9m0937","country_code":"AT","type":"education","lineage":["https://openalex.org/I4210166741"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Faisal Shehzad","raw_affiliation_strings":["University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria"],"raw_orcid":"https://orcid.org/0000-0001-6239-8198","affiliations":[{"raw_affiliation_string":"University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria","institution_ids":["https://openalex.org/I4210166741"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067984355","display_name":"Timo Breuer","orcid":"https://orcid.org/0000-0002-1765-2449"},"institutions":[{"id":"https://openalex.org/I102520234","display_name":"TH K\u00f6ln - University of Applied Sciences","ror":"https://ror.org/014nnvj65","country_code":"DE","type":"education","lineage":["https://openalex.org/I102520234"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Timo Breuer","raw_affiliation_strings":["TH K\u00f6ln (University of Applied Sciences), K\u00f6ln, Germany"],"raw_orcid":"https://orcid.org/0000-0002-1765-2449","affiliations":[{"raw_affiliation_string":"TH K\u00f6ln (University of Applied Sciences), K\u00f6ln, Germany","institution_ids":["https://openalex.org/I102520234"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078639165","display_name":"Maria Maistro","orcid":"https://orcid.org/0000-0002-7001-4817"},"institutions":[{"id":"https://openalex.org/I124055696","display_name":"University of Copenhagen","ror":"https://ror.org/035b05819","country_code":"DK","type":"education","lineage":["https://openalex.org/I124055696"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Maria Maistro","raw_affiliation_strings":["University of Copenhagen, Copenhagen, Denmark"],"raw_orcid":"https://orcid.org/0000-0002-7001-4817","affiliations":[{"raw_affiliation_string":"University of Copenhagen, Copenhagen, Denmark","institution_ids":["https://openalex.org/I124055696"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005545405","display_name":"Dietmar Jannach","orcid":"https://orcid.org/0000-0002-4698-8507"},"institutions":[{"id":"https://openalex.org/I4210166741","display_name":"University of Klagenfurt","ror":"https://ror.org/05q9m0937","country_code":"AT","type":"education","lineage":["https://openalex.org/I4210166741"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Dietmar Jannach","raw_affiliation_strings":["University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria"],"raw_orcid":"https://orcid.org/0000-0002-4698-8507","affiliations":[{"raw_affiliation_string":"University of Klagenfurt, Klagenfurt am W\u00f6rthersee, Austria","institution_ids":["https://openalex.org/I4210166741"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5071874921"],"corresponding_institution_ids":["https://openalex.org/I4210166741"],"apc_list":null,"apc_paid":null,"fwci":2.9051,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.92726485,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"884","last_page":"893"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9929999709129333,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8375691175460815},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7719061374664307},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6007617115974426},{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.48888319730758667},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.2695666551589966},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.09233880043029785},{"id":"https://openalex.org/keywords/chemistry","display_name":"Chemistry","score":0.059466421604156494}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8375691175460815},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7719061374664307},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6007617115974426},{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.48888319730758667},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.2695666551589966},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.09233880043029785},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.059466421604156494},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3705328.3748157","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748157","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:pure.atira.dk:openaire_cris_publications/7e6762da-8b5c-47ae-b34f-f33291b7e6ac","is_oa":true,"landing_page_url":"https://researchprofiles.ku.dk/da/publications/7e6762da-8b5c-47ae-b34f-f33291b7e6ac","pdf_url":"https://curis.ku.dk/ws/files/534751153/3705328.3748157.pdf","source":{"id":"https://openalex.org/S4306401983","display_name":"Research at the University of Copenhagen (University of Copenhagen)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I124055696","host_organization_name":"University of Copenhagen","host_organization_lineage":["https://openalex.org/I124055696"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Shehzad , F , Breuer , T , Maistro , M & Jannach , D 2025 , \"We Share Our Code Online\" : Why This Is Not Enough to Ensure Reproducibility and Progress in Recommender Systems Research . in RecSys '25: Proceedings of the Nineteenth ACM Conference on Recommender Systems . Association for Computing Machinery, Inc. , pp. 884-893 , RecSys '25: 19th ACM Conference on Recommender Systems RecSys 2025 , Prague , Czech Republic , 22/09/2025 . https://doi.org/10.1145/3705328.3748157","raw_type":"contributionToPeriodical"}],"best_oa_location":{"id":"doi:10.1145/3705328.3748157","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748157","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":52,"referenced_works":["https://openalex.org/W1574680346","https://openalex.org/W1965355809","https://openalex.org/W1981575590","https://openalex.org/W1991664944","https://openalex.org/W2054141820","https://openalex.org/W2078488685","https://openalex.org/W2080356852","https://openalex.org/W2161351857","https://openalex.org/W2309034743","https://openalex.org/W2336806308","https://openalex.org/W2788572835","https://openalex.org/W2883256923","https://openalex.org/W2891211322","https://openalex.org/W2957191877","https://openalex.org/W2982267219","https://openalex.org/W2987577970","https://openalex.org/W3044893918","https://openalex.org/W3088444111","https://openalex.org/W3088694469","https://openalex.org/W3088777230","https://openalex.org/W3088831118","https://openalex.org/W3093891819","https://openalex.org/W3101064725","https://openalex.org/W3103776775","https://openalex.org/W3105114834","https://openalex.org/W3125645198","https://openalex.org/W3134063549","https://openalex.org/W3154190849","https://openalex.org/W3154199194","https://openalex.org/W3164378906","https://openalex.org/W3192665785","https://openalex.org/W3205774243","https://openalex.org/W3208227120","https://openalex.org/W3211225893","https://openalex.org/W3211255867","https://openalex.org/W4229000037","https://openalex.org/W4283205337","https://openalex.org/W4285806565","https://openalex.org/W4376873982","https://openalex.org/W4378945316","https://openalex.org/W4384648390","https://openalex.org/W4385373885","https://openalex.org/W4385373976","https://openalex.org/W4386729058","https://openalex.org/W4386729258","https://openalex.org/W4387999295","https://openalex.org/W4402896964","https://openalex.org/W4403220390","https://openalex.org/W4403220878","https://openalex.org/W4405867014","https://openalex.org/W4408786097","https://openalex.org/W4409415146"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W4324138256"],"abstract_inverted_index":{"Issues":[0],"with":[1],"reproducibility":[2,93,104,219,227],"have":[3],"been":[4],"identified":[5],"as":[6],"a":[7,52,111,170],"major":[8],"factor":[9],"hampering":[10],"progress":[11,231],"in":[12,94,97,129,144,166,185,232],"recommender":[13,98],"systems":[14],"research.":[15],"In":[16,44],"response,":[17],"researchers":[18],"increasingly":[19],"share":[20],"the":[21,27,31,34,47,57,60,67,73,76,79,90,103,115,124,134,136,139,150,180,186,193,197,200,209,215],"code":[22,32,126,137],"of":[23,29,33,59,78,92,106,133,138,149,199,218],"their":[24],"models.":[25],"However,":[26],"provision":[28],"only":[30,143,176],"proposed":[35],"model":[36,54,125],"is":[37,50,55,64,70,87,127,141,163,173,189],"usually":[38],"not":[39,164,175,213],"sufficient":[40],"to":[41,88,178,229],"ensure":[42,230],"reproducibility.":[43],"many":[45],"works,":[46],"central":[48],"claim":[49],"that":[51,66,208],"new":[53],"advancing":[56],"state":[58,198],"art.":[61],"Thus,":[62],"it":[63,172,188],"crucial":[65],"entire":[68],"experiment":[69],"reproducible,":[71],"including":[72],"configuration":[74],"and":[75,157],"results":[77,120],"considered":[80],"baselines.":[81],"With":[82],"this":[83,233],"work,":[84],"our":[85],"goal":[86],"gauge":[89],"level":[91,105,217],"algorithms":[95],"research":[96,210],"systems.":[99],"We":[100,221],"systematically":[101],"analyzed":[102],"65":[107],"papers":[108],"published":[109],"at":[110],"top-ranked":[112],"conference":[113],"during":[114],"last":[116],"three":[117],"years.":[118],"Our":[119],"are":[121,152],"sobering.":[122],"While":[123],"shared":[128],"about":[130],"two":[131],"thirds":[132],"papers,":[135,187],"baselines":[140,151],"provided":[142],"eight":[145],"cases.":[146],"The":[147],"hyperparameters":[148],"reported":[153,184],"even":[154],"less":[155],"frequently,":[156],"how":[158],"these":[159],"were":[160,202],"exactly":[161],"determined":[162],"explained":[165],"any":[167],"paper.":[168],"As":[169],"result,":[171],"commonly":[174],"impossible":[177],"reproduce":[179],"full":[181],"result":[182],"tables":[183],"also":[190],"unclear":[191],"if":[192],"claimed":[194],"improvements":[195],"over":[196],"art":[201],"actually":[203],"achieved.":[204],"Overall,":[205],"we":[206],"conclude":[207],"community":[211],"has":[212],"reached":[214],"required":[216],"yet.":[220],"therefore":[222],"call":[223],"for":[224],"more":[225],"rigorous":[226],"standards":[228],"field.":[234]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}