{"id":"https://openalex.org/W2913491198","doi":"https://doi.org/10.1145/3289600.3291027","title":"Offline Evaluation to Make Decisions About PlaylistRecommendation Algorithms","display_name":"Offline Evaluation to Make Decisions About PlaylistRecommendation Algorithms","publication_year":2019,"publication_date":"2019-01-30","ids":{"openalex":"https://openalex.org/W2913491198","doi":"https://doi.org/10.1145/3289600.3291027","mag":"2913491198"},"language":"en","primary_location":{"id":"doi:10.1145/3289600.3291027","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3289600.3291027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073739285","display_name":"Alois Gruson","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Alois Gruson","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5020257829","display_name":"Praveen Chandar","orcid":"https://orcid.org/0009-0008-2199-2631"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Praveen Chandar","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068842407","display_name":"Christophe Charbuillet","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Christophe Charbuillet","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037761875","display_name":"James O. McInerney","orcid":"https://orcid.org/0000-0003-1885-2503"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"James McInerney","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065241086","display_name":"Samantha Hansen","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samantha Hansen","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058813791","display_name":"Damien Tardieu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Damien Tardieu","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078123101","display_name":"Ben Carterette","orcid":"https://orcid.org/0000-0001-9538-047X"},"institutions":[{"id":"https://openalex.org/I4210122154","display_name":"Photon Spot (United States)","ror":"https://ror.org/01yxc0v75","country_code":"US","type":"company","lineage":["https://openalex.org/I4210122154"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ben Carterette","raw_affiliation_strings":["Spotify, New York, NY, USA"],"affiliations":[{"raw_affiliation_string":"Spotify, New York, NY, USA","institution_ids":["https://openalex.org/I4210122154"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5073739285"],"corresponding_institution_ids":["https://openalex.org/I4210122154"],"apc_list":null,"apc_paid":null,"fwci":10.695,"has_fulltext":false,"cited_by_count":135,"citation_normalized_percentile":{"value":0.98436446,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"420","last_page":"428"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9983000159263611,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11704","display_name":"Mobile Crowdsensing and Crowdsourcing","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8334544897079468},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7256072163581848},{"id":"https://openalex.org/keywords/counterfactual-thinking","display_name":"Counterfactual thinking","score":0.6589462161064148},{"id":"https://openalex.org/keywords/online-and-offline","display_name":"Online and offline","score":0.5534300804138184},{"id":"https://openalex.org/keywords/test","display_name":"Test (biology)","score":0.46833041310310364},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.44259369373321533},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4280531406402588},{"id":"https://openalex.org/keywords/online-algorithm","display_name":"Online algorithm","score":0.42476335167884827},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.36296194791793823},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.257293164730072}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8334544897079468},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7256072163581848},{"id":"https://openalex.org/C108650721","wikidata":"https://www.wikidata.org/wiki/Q1783253","display_name":"Counterfactual thinking","level":2,"score":0.6589462161064148},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.5534300804138184},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.46833041310310364},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.44259369373321533},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4280531406402588},{"id":"https://openalex.org/C196921405","wikidata":"https://www.wikidata.org/wiki/Q786431","display_name":"Online algorithm","level":2,"score":0.42476335167884827},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.36296194791793823},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.257293164730072},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3289600.3291027","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3289600.3291027","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twelfth ACM International Conference on Web Search and Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.46000000834465027}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W135190683","https://openalex.org/W298464722","https://openalex.org/W1483313504","https://openalex.org/W1556219185","https://openalex.org/W1967787801","https://openalex.org/W1971040550","https://openalex.org/W1974360117","https://openalex.org/W1982530130","https://openalex.org/W1998427280","https://openalex.org/W2001947543","https://openalex.org/W2026784708","https://openalex.org/W2037140704","https://openalex.org/W2055598786","https://openalex.org/W2057744254","https://openalex.org/W2069870183","https://openalex.org/W2078488685","https://openalex.org/W2094790959","https://openalex.org/W2097638479","https://openalex.org/W2099213975","https://openalex.org/W2110228583","https://openalex.org/W2122124659","https://openalex.org/W2132314908","https://openalex.org/W2138216384","https://openalex.org/W2138909795","https://openalex.org/W2144807535","https://openalex.org/W2150291618","https://openalex.org/W2150886314","https://openalex.org/W2155106456","https://openalex.org/W2155587858","https://openalex.org/W2156541733","https://openalex.org/W2161676175","https://openalex.org/W2163162311","https://openalex.org/W2188353343","https://openalex.org/W2207136466","https://openalex.org/W2219888463","https://openalex.org/W2340526403","https://openalex.org/W2402441596","https://openalex.org/W2464171116","https://openalex.org/W2507134384","https://openalex.org/W2511414107","https://openalex.org/W2575238773","https://openalex.org/W2594908799","https://openalex.org/W2618462075","https://openalex.org/W2784068709","https://openalex.org/W2799155063","https://openalex.org/W2893370267","https://openalex.org/W2913059114","https://openalex.org/W2962694783","https://openalex.org/W3098679278","https://openalex.org/W3099420497","https://openalex.org/W4233471163","https://openalex.org/W4293653493","https://openalex.org/W6633499030","https://openalex.org/W6713202564"],"related_works":["https://openalex.org/W3201448254","https://openalex.org/W4286970243","https://openalex.org/W2066431708","https://openalex.org/W4384133558","https://openalex.org/W3025615835","https://openalex.org/W173210993","https://openalex.org/W2390660599","https://openalex.org/W3028847759","https://openalex.org/W2393688264","https://openalex.org/W3042122676"],"abstract_inverted_index":{"Evaluating":[0],"algorithmic":[1],"recommendations":[2],"is":[3,67,78,90,150],"an":[4,19,236],"important,":[5],"but":[6,46],"difficult,":[7],"problem.":[8],"Evaluations":[9],"conducted":[10],"offline":[11,117,157,171,200,221,237],"using":[12],"data":[13],"collected":[14],"from":[15,24,27],"user":[16,29,60,76,187],"interactions":[17],"with":[18,98,136,248],"online":[20,96,145,173,246],"system":[21],"often":[22],"suffer":[23],"biases":[25],"arising":[26],"the":[28,32,59,142,214,241],"interface":[30],"or":[31,105],"recommendation":[33,99,179],"engine.":[34],"Online":[35],"evaluation":[36,158,174,238],"(A/B":[37],"testing)":[38],"can":[39,50,234],"more":[40,69,203],"easily":[41],"address":[42],"problems":[43],"of":[44,56,95,144,155,170,183,213],"bias,":[45],"depending":[47],"on":[48,86,119,130,217],"setting":[49],"be":[51],"time-consuming":[52],"and":[53,111,137,172,230],"incur":[54],"risk":[55],"negatively":[57],"impacting":[58],"experience,":[61],"not":[62,79,126],"to":[63,73,91,115,140,152,185,196,211,226,239],"mention":[64],"that":[65,101,232],"it":[66],"generally":[68],"difficult":[70],"when":[71],"access":[72],"a":[74,168,177,186],"large":[75],"base":[77],"taken":[80],"as":[81],"granted.":[82],"A":[83],"compromise":[84],"based":[85],"\\em":[87],"counterfactual":[88],"analysis":[89],"present":[92,167],"some":[93],"subset":[94],"users":[97],"results":[100,143,175,207],"have":[102],"been":[103],"randomized":[104],"otherwise":[106],"manipulated,":[107],"log":[108],"their":[109],"interactions,":[110],"then":[112],"use":[113],"those":[114],"de-bias":[116],"evaluations":[118],"historical":[120],"data.":[121],"However,":[122],"previous":[123,215],"work":[124,165,216],"does":[125],"offer":[127],"clear":[128],"conclusions":[129],"how":[131],"well":[132,225],"such":[133],"methods":[134],"correlate":[135,224],"are":[138],"able":[139],"predict":[141],"A/B":[146,227],"tests.":[147],"Understanding":[148],"this":[149,164,218],"crucial":[151],"widespread":[153],"adoption":[154],"new":[156],"techniques":[159],"in":[160],"recommender":[161],"systems.":[162],"In":[163],"we":[166,233],"comparison":[169],"for":[176,189,202,245],"particular":[178],"problem:":[180],"recommending":[181],"playlists":[182],"tracks":[184],"looking":[188],"music.":[190],"We":[191],"describe":[192],"two":[193],"different":[194],"ways":[195],"think":[197],"about":[198],"de-biasing":[199],"collections":[201],"accurate":[204],"evaluation.":[205],"Our":[206],"show":[208],"that,":[209],"contrary":[210],"much":[212],"topic,":[219],"properly-conducted":[220],"experiments":[222],"do":[223],"test":[228],"results,":[229],"moreover":[231],"expect":[235],"identify":[240],"best":[242],"candidate":[243],"systems":[244],"testing":[247],"high":[249],"probability.":[250]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":16},{"year":2024,"cited_by_count":45},{"year":2023,"cited_by_count":20},{"year":2022,"cited_by_count":15},{"year":2021,"cited_by_count":17},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":8},{"year":2012,"cited_by_count":1}],"updated_date":"2026-03-12T08:34:05.389933","created_date":"2025-10-10T00:00:00"}