{"id":"https://openalex.org/W4414034908","doi":"https://doi.org/10.1145/3705328.3748086","title":"On the Reliability of Sampling Strategies in Offline Recommender Evaluation","display_name":"On the Reliability of Sampling Strategies in Offline Recommender Evaluation","publication_year":2025,"publication_date":"2025-09-06","ids":{"openalex":"https://openalex.org/W4414034908","doi":"https://doi.org/10.1145/3705328.3748086"},"language":"en","primary_location":{"id":"doi:10.1145/3705328.3748086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748086","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3705328.3748086","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3705328.3748086","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5023961364","display_name":"Bruno Laporais Pereira","orcid":"https://orcid.org/0009-0004-2857-2913"},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]}],"countries":["BR"],"is_corresponding":true,"raw_author_name":"Bruno L. Pereira","raw_affiliation_strings":["Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil"],"raw_orcid":"https://orcid.org/0009-0004-2857-2913","affiliations":[{"raw_affiliation_string":"Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil","institution_ids":["https://openalex.org/I110200422"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040472816","display_name":"Alan Said","orcid":"https://orcid.org/0000-0002-2929-0529"},"institutions":[{"id":"https://openalex.org/I881427289","display_name":"University of Gothenburg","ror":"https://ror.org/01tm6cn81","country_code":"SE","type":"education","lineage":["https://openalex.org/I881427289"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Alan Said","raw_affiliation_strings":["University of Gothenburg, Gothenburg, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-2929-0529","affiliations":[{"raw_affiliation_string":"University of Gothenburg, Gothenburg, Sweden","institution_ids":["https://openalex.org/I881427289"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025385531","display_name":"Rodrygo L. T. Santos","orcid":"https://orcid.org/0000-0003-2921-8444"},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]}],"countries":["BR"],"is_corresponding":false,"raw_author_name":"Rodrygo L. T. Santos","raw_affiliation_strings":["Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil"],"raw_orcid":"https://orcid.org/0000-0003-2921-8444","affiliations":[{"raw_affiliation_string":"Universidade Federal de Minas Gerais, Belo Horizonte, MG, Brazil","institution_ids":["https://openalex.org/I110200422"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5023961364"],"corresponding_institution_ids":["https://openalex.org/I110200422"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.34211827,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"360","last_page":"369"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8017666935920715},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7273135781288147},{"id":"https://openalex.org/keywords/reliability","display_name":"Reliability (semiconductor)","score":0.6791379451751709},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.5412471294403076},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.3409428000450134},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.28433358669281006},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.09232783317565918},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07777410745620728}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8017666935920715},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7273135781288147},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.6791379451751709},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.5412471294403076},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3409428000450134},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.28433358669281006},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.09232783317565918},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07777410745620728},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C163258240","wikidata":"https://www.wikidata.org/wiki/Q25342","display_name":"Power (physics)","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3705328.3748086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748086","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3705328.3748086","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2508.05398","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2508.05398","pdf_url":"https://arxiv.org/pdf/2508.05398","source":{"id":"https://openalex.org/S4393918464","display_name":"ArXiv.org","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"}],"best_oa_location":{"id":"doi:10.1145/3705328.3748086","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3705328.3748086","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3705328.3748086","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Nineteenth ACM Conference on Recommender Systems","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"},{"id":"https://openalex.org/F4320322980","display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais","ror":"https://ror.org/00nc55f03"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4414034908.pdf","grobid_xml":"https://content.openalex.org/works/W4414034908.grobid-xml"},"referenced_works_count":48,"referenced_works":["https://openalex.org/W1886704267","https://openalex.org/W1992665562","https://openalex.org/W1994389483","https://openalex.org/W1998427280","https://openalex.org/W2005415325","https://openalex.org/W2042281163","https://openalex.org/W2045745608","https://openalex.org/W2080371694","https://openalex.org/W2116008435","https://openalex.org/W2150886314","https://openalex.org/W2223495811","https://openalex.org/W2784068709","https://openalex.org/W2799048248","https://openalex.org/W2892923641","https://openalex.org/W2897844661","https://openalex.org/W2950627632","https://openalex.org/W2972520532","https://openalex.org/W2992252352","https://openalex.org/W2998534896","https://openalex.org/W3015088447","https://openalex.org/W3034503922","https://openalex.org/W3080781457","https://openalex.org/W3081170586","https://openalex.org/W3089147479","https://openalex.org/W3089214483","https://openalex.org/W3092103025","https://openalex.org/W3098679278","https://openalex.org/W3099420497","https://openalex.org/W3177788761","https://openalex.org/W3185773347","https://openalex.org/W3201149665","https://openalex.org/W4224318144","https://openalex.org/W4233760227","https://openalex.org/W4239728164","https://openalex.org/W4281848017","https://openalex.org/W4283018239","https://openalex.org/W4286222818","https://openalex.org/W4288079520","https://openalex.org/W4292092796","https://openalex.org/W4292419518","https://openalex.org/W4292423901","https://openalex.org/W4296591809","https://openalex.org/W4386729510","https://openalex.org/W4386730264","https://openalex.org/W4386977429","https://openalex.org/W4387968576","https://openalex.org/W4396821616","https://openalex.org/W4401856737"],"related_works":["https://openalex.org/W2033512842","https://openalex.org/W4233600955","https://openalex.org/W4322734194","https://openalex.org/W3116237489","https://openalex.org/W4404996554","https://openalex.org/W2913665393","https://openalex.org/W2369695847","https://openalex.org/W3005535424","https://openalex.org/W2994319598","https://openalex.org/W2047067935"],"abstract_inverted_index":{"Offline":[0],"evaluation":[1,42,164],"plays":[2],"a":[3,46,111],"central":[4],"role":[5],"in":[6],"benchmarking":[7],"recommender":[8],"systems":[9],"when":[10,41,159],"online":[11],"testing":[12],"is":[13,18,43],"impractical":[14],"or":[15,87],"risky.However,":[16],"it":[17],"susceptible":[19],"to":[20,61,78,89],"two":[21],"key":[22],"sources":[23],"of":[24,48,100,108,128],"bias:":[25],"exposure":[26,85,122,148],"bias,":[27,39,64],"where":[28],"users":[29],"only":[30],"interact":[31],"with":[32,142,154],"items":[33,50],"they":[34],"are":[35,66],"shown,":[36],"and":[37,102,124,150,160,166,176],"sampling":[38,63,103,130,135,162],"introduced":[40],"performed":[44],"on":[45,69],"subset":[47],"logged":[49,71],"rather":[51,73],"than":[52,74],"the":[53,106,126],"full":[54,143],"catalog.While":[55],"prior":[56],"work":[57],"has":[58],"proposed":[59],"methods":[60],"mitigate":[62],"these":[65],"typically":[67],"assessed":[68],"fixed":[70],"datasets":[72],"for":[75,170],"their":[76],"ability":[77],"support":[79],"reliable":[80],"model":[81,138],"comparisons":[82],"under":[83,147],"varying":[84],"conditions":[86],"relative":[88],"true":[90],"user":[91],"preferences.In":[92],"this":[93],"paper,":[94],"we":[95,118],"investigate":[96],"how":[97,161],"different":[98],"combinations":[99],"logging":[101],"choices":[104],"affect":[105],"reliability":[107,127],"offline":[109,178],"evaluation.Using":[110],"fully":[112],"observed":[113],"dataset":[114],"as":[115],"ground":[116,155],"truth,":[117],"systematically":[119],"simulate":[120],"diverse":[121],"biases":[123],"assess":[125],"common":[129],"strategies":[131,172],"along":[132],"four":[133],"dimensions:":[134],"resolution":[136],"(recommender":[137],"separability),":[139],"fidelity":[140],"(agreement":[141],"evaluation),":[144],"robustness":[145],"(stability":[146],"bias),":[149],"predictive":[151],"power":[152],"(alignment":[153],"truth).Our":[156],"findings":[157],"highlight":[158],"distorts":[163],"outcomes":[165],"offer":[167],"practical":[168],"guidance":[169],"selecting":[171],"that":[173],"yield":[174],"faithful":[175],"robust":[177],"comparisons.":[179]},"counts_by_year":[],"updated_date":"2025-11-06T06:51:31.235846","created_date":"2025-10-10T00:00:00"}