{"id":"https://openalex.org/W4387999295","doi":"https://doi.org/10.1145/3629170","title":"Exploring the Landscape of Recommender Systems Evaluation: Practices and Perspectives","display_name":"Exploring the Landscape of Recommender Systems Evaluation: Practices and Perspectives","publication_year":2023,"publication_date":"2023-10-28","ids":{"openalex":"https://openalex.org/W4387999295","doi":"https://doi.org/10.1145/3629170"},"language":"en","primary_location":{"id":"doi:10.1145/3629170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3629170","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3629170","source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3629170","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033035188","display_name":"Christine Bauer","orcid":"https://orcid.org/0000-0001-5724-1137"},"institutions":[{"id":"https://openalex.org/I182212641","display_name":"University of Salzburg","ror":"https://ror.org/05gs8cd61","country_code":"AT","type":"education","lineage":["https://openalex.org/I182212641"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Christine Bauer","raw_affiliation_strings":["Paris Lodron University Salzburg, Salzburg, Austria"],"affiliations":[{"raw_affiliation_string":"Paris Lodron University Salzburg, Salzburg, Austria","institution_ids":["https://openalex.org/I182212641"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070525687","display_name":"Eva Zangerle","orcid":"https://orcid.org/0000-0003-3195-8273"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":false,"raw_author_name":"Eva Zangerle","raw_affiliation_strings":["University of Innsbruck, Innsbruck, Austria"],"affiliations":[{"raw_affiliation_string":"University of Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040472816","display_name":"Alan Said","orcid":"https://orcid.org/0000-0002-2929-0529"},"institutions":[{"id":"https://openalex.org/I881427289","display_name":"University of Gothenburg","ror":"https://ror.org/01tm6cn81","country_code":"SE","type":"education","lineage":["https://openalex.org/I881427289"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Alan Said","raw_affiliation_strings":["University of Gothenburg, Sweden"],"affiliations":[{"raw_affiliation_string":"University of Gothenburg, Sweden","institution_ids":["https://openalex.org/I881427289"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5033035188"],"corresponding_institution_ids":["https://openalex.org/I182212641"],"apc_list":null,"apc_paid":null,"fwci":6.9447,"has_fulltext":true,"cited_by_count":15,"citation_normalized_percentile":{"value":0.97016545,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":99},"biblio":{"volume":"2","issue":"1","first_page":"1","last_page":"31"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9947999715805054,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9772999882698059,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/movielens","display_name":"MovieLens","score":0.8729085326194763},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8148176670074463},{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.8136026859283447},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6597222685813904},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.5527997612953186},{"id":"https://openalex.org/keywords/scope","display_name":"Scope (computer science)","score":0.5454179048538208},{"id":"https://openalex.org/keywords/precision-and-recall","display_name":"Precision and recall","score":0.4905835688114166},{"id":"https://openalex.org/keywords/collaborative-filtering","display_name":"Collaborative filtering","score":0.48135095834732056},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.42361265420913696},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3382530212402344},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3373796343803406},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.28257712721824646}],"concepts":[{"id":"https://openalex.org/C2776156558","wikidata":"https://www.wikidata.org/wiki/Q4353746","display_name":"MovieLens","level":4,"score":0.8729085326194763},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8148176670074463},{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.8136026859283447},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6597222685813904},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.5527997612953186},{"id":"https://openalex.org/C2778012447","wikidata":"https://www.wikidata.org/wiki/Q1034415","display_name":"Scope (computer science)","level":2,"score":0.5454179048538208},{"id":"https://openalex.org/C81669768","wikidata":"https://www.wikidata.org/wiki/Q2359161","display_name":"Precision and recall","level":2,"score":0.4905835688114166},{"id":"https://openalex.org/C21569690","wikidata":"https://www.wikidata.org/wiki/Q94702","display_name":"Collaborative filtering","level":3,"score":0.48135095834732056},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.42361265420913696},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3382530212402344},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3373796343803406},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28257712721824646},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3629170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3629170","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3629170","source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3629170","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3629170","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3629170","source":{"id":"https://openalex.org/S4387284325","display_name":"ACM Transactions on Recommender Systems","issn_l":"2770-6699","issn":["2770-6699"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Recommender Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4300000071525574,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[{"id":"https://openalex.org/G1837357248","display_name":null,"funder_award_id":"P33526","funder_id":"https://openalex.org/F4320321181","funder_display_name":"Austrian Science Fund"}],"funders":[{"id":"https://openalex.org/F4320321030","display_name":"VINNOVA","ror":"https://ror.org/01kd5m353"},{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387999295.pdf","grobid_xml":"https://content.openalex.org/works/W4387999295.grobid-xml"},"referenced_works_count":103,"referenced_works":["https://openalex.org/W94637519","https://openalex.org/W299870200","https://openalex.org/W1015675232","https://openalex.org/W1483624827","https://openalex.org/W1971040550","https://openalex.org/W2001259128","https://openalex.org/W2009205701","https://openalex.org/W2050113838","https://openalex.org/W2090679992","https://openalex.org/W2110953678","https://openalex.org/W2150886314","https://openalex.org/W2153274408","https://openalex.org/W2155912844","https://openalex.org/W2157881433","https://openalex.org/W2210549170","https://openalex.org/W2219888463","https://openalex.org/W2492959872","https://openalex.org/W2513929965","https://openalex.org/W2739587313","https://openalex.org/W2740253077","https://openalex.org/W2741456980","https://openalex.org/W2749464641","https://openalex.org/W2750446834","https://openalex.org/W2784068709","https://openalex.org/W2791379569","https://openalex.org/W2799048248","https://openalex.org/W2803314776","https://openalex.org/W2892002792","https://openalex.org/W2892888989","https://openalex.org/W2893160345","https://openalex.org/W2906896588","https://openalex.org/W2908515202","https://openalex.org/W2913853152","https://openalex.org/W2944441143","https://openalex.org/W2953703039","https://openalex.org/W2957191877","https://openalex.org/W2963367478","https://openalex.org/W2963655167","https://openalex.org/W2971196067","https://openalex.org/W2972941122","https://openalex.org/W2981030381","https://openalex.org/W2984100107","https://openalex.org/W2987577970","https://openalex.org/W3021048486","https://openalex.org/W3034491755","https://openalex.org/W3034718811","https://openalex.org/W3040911419","https://openalex.org/W3044963235","https://openalex.org/W3080781457","https://openalex.org/W3081170586","https://openalex.org/W3088444111","https://openalex.org/W3088869197","https://openalex.org/W3088872368","https://openalex.org/W3089147479","https://openalex.org/W3098217413","https://openalex.org/W3099420497","https://openalex.org/W3101793783","https://openalex.org/W3104589861","https://openalex.org/W3105114834","https://openalex.org/W3122085188","https://openalex.org/W3125645198","https://openalex.org/W3129660887","https://openalex.org/W3134041167","https://openalex.org/W3134063549","https://openalex.org/W3152634418","https://openalex.org/W3154190849","https://openalex.org/W3155375847","https://openalex.org/W3156627302","https://openalex.org/W3164231466","https://openalex.org/W3168154995","https://openalex.org/W3171860698","https://openalex.org/W3175111331","https://openalex.org/W3177788761","https://openalex.org/W3185773347","https://openalex.org/W3190132430","https://openalex.org/W3197958031","https://openalex.org/W3199097559","https://openalex.org/W3200267675","https://openalex.org/W3200799482","https://openalex.org/W3201149665","https://openalex.org/W3201265114","https://openalex.org/W3205774243","https://openalex.org/W3208543775","https://openalex.org/W3210115933","https://openalex.org/W3210737511","https://openalex.org/W3210948593","https://openalex.org/W3214575884","https://openalex.org/W3217771118","https://openalex.org/W4206511970","https://openalex.org/W4212843742","https://openalex.org/W4213113302","https://openalex.org/W4220840776","https://openalex.org/W4226037318","https://openalex.org/W4283587264","https://openalex.org/W4284699674","https://openalex.org/W4284701291","https://openalex.org/W4285091076","https://openalex.org/W4285806565","https://openalex.org/W4292092796","https://openalex.org/W4292596048","https://openalex.org/W4296604468","https://openalex.org/W4297614008","https://openalex.org/W6775204245"],"related_works":["https://openalex.org/W2355698112","https://openalex.org/W2022984797","https://openalex.org/W4394818607","https://openalex.org/W2986679525","https://openalex.org/W2797500822","https://openalex.org/W2794458286","https://openalex.org/W4205822456","https://openalex.org/W4299358966","https://openalex.org/W2537367858","https://openalex.org/W2942098650"],"abstract_inverted_index":{"Recommender":[0],"systems":[1,87],"research":[2,27,81,182],"and":[3,68,91,155,210],"practice":[4],"are":[5,95,120,126,147,160,174],"fast-developing":[6],"topics":[7],"with":[8,100,199],"growing":[9],"adoption":[10],"in":[11,42,60,80,98,128,162,194],"a":[12,38,113,130,136,164,195,206],"wide":[13],"variety":[14],"of":[15,26,33,85,192,202],"information":[16],"access":[17],"scenarios.":[18],"In":[19],"this":[20],"article,":[21],"we":[22,44,58,109],"present":[23],"an":[24,105],"overview":[25],"specifically":[28],"focused":[29,188],"on":[30,53,61,82,185,189,205],"the":[31,54,62,65,69,76,83,141,181,190,200],"evaluation":[32,84,186,193],"recommender":[34,86],"systems.":[35],"We":[36,134],"perform":[37],"systematic":[39],"literature":[40],"review,":[41],"which":[43],"analyze":[45],"57":[46],"papers":[47,132],"spanning":[48],"six":[49],"years":[50],"(2017\u20132022).":[51],"Focusing":[52],"processes":[55],"surrounding":[56],"evaluation,":[57],"dial":[59],"methods":[63],"applied,":[64],"datasets":[66,115,125],"utilized,":[67],"metrics":[70,146],"used.":[71],"Our":[72,177],"study":[73],"shows":[74,179],"that":[75,92,111,171,180],"predominant":[77],"experiment":[78],"type":[79],"is":[88],"offline":[89,106],"experimentation":[90,102],"online":[93],"evaluations":[94],"primarily":[96],"used":[97,127,149,161],"combination":[99],"other":[101],"methods,":[103],"e.g.,":[104],"experiment.":[107],"Furthermore,":[108],"find":[110],"only":[112,129,163],"few":[114,131,145,165,207],"(MovieLens,":[116],"Amazon":[117],"review":[118,169],"dataset)":[119],"widely":[121,148],"used,":[122],"while":[123,157],"many":[124,158],"each.":[133],"observe":[135],"similar":[137],"scenario":[138],"when":[139],"analyzing":[140],"employed":[142],"performance":[143],"metrics\u2014a":[144],"(precision,":[150],"normalized":[151],"Discounted":[152],"Cumulative":[153],"Gain,":[154],"Recall),":[156],"others":[159],"papers.":[166],"Overall,":[167],"our":[168],"indicates":[170],"beyond-accuracy":[172],"qualities":[173],"rarely":[175],"assessed.":[176],"analysis":[178],"community":[183],"working":[184],"has":[187],"development":[191],"rather":[196],"narrow":[197],"scope,":[198],"majority":[201],"experiments":[203],"focusing":[204],"metrics,":[208],"datasets,":[209],"methods.":[211]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":4}],"updated_date":"2026-03-13T16:22:10.518609","created_date":"2025-10-10T00:00:00"}