{"id":"https://openalex.org/W4292092796","doi":"https://doi.org/10.1145/3556536","title":"Evaluating Recommender Systems: Survey and Framework","display_name":"Evaluating Recommender Systems: Survey and Framework","publication_year":2022,"publication_date":"2022-08-17","ids":{"openalex":"https://openalex.org/W4292092796","doi":"https://doi.org/10.1145/3556536"},"language":"en","primary_location":{"id":"doi:10.1145/3556536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3556536","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3556536","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"type":"review","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3556536","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070525687","display_name":"Eva Zangerle","orcid":"https://orcid.org/0000-0003-3195-8273"},"institutions":[{"id":"https://openalex.org/I190249584","display_name":"Universit\u00e4t Innsbruck","ror":"https://ror.org/054pv6659","country_code":"AT","type":"education","lineage":["https://openalex.org/I190249584"]}],"countries":["AT"],"is_corresponding":true,"raw_author_name":"Eva Zangerle","raw_affiliation_strings":["Universit\u00e4t Innsbruck, Innsbruck, Austria"],"affiliations":[{"raw_affiliation_string":"Universit\u00e4t Innsbruck, Innsbruck, Austria","institution_ids":["https://openalex.org/I190249584"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5033035188","display_name":"Christine Bauer","orcid":"https://orcid.org/0000-0001-5724-1137"},"institutions":[{"id":"https://openalex.org/I193662353","display_name":"Utrecht University","ror":"https://ror.org/04pp8hn57","country_code":"NL","type":"education","lineage":["https://openalex.org/I193662353"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Christine Bauer","raw_affiliation_strings":["Utrecht University, Utrecht, Princetonplein, The Netherlands"],"affiliations":[{"raw_affiliation_string":"Utrecht University, Utrecht, Princetonplein, The Netherlands","institution_ids":["https://openalex.org/I193662353"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5070525687"],"corresponding_institution_ids":["https://openalex.org/I190249584"],"apc_list":null,"apc_paid":null,"fwci":66.6432,"has_fulltext":true,"cited_by_count":229,"citation_normalized_percentile":{"value":0.99957063,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"55","issue":"8","first_page":"1","last_page":"38"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9907000064849854,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9900000095367432,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.9365315437316895},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.9072532653808594},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.604961097240448},{"id":"https://openalex.org/keywords/field","display_name":"Field (mathematics)","score":0.567987859249115},{"id":"https://openalex.org/keywords/foundation","display_name":"Foundation (evidence)","score":0.42774778604507446},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.42185789346694946},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.39803895354270935},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3198421001434326},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.25978147983551025}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.9365315437316895},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.9072532653808594},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.604961097240448},{"id":"https://openalex.org/C9652623","wikidata":"https://www.wikidata.org/wiki/Q190109","display_name":"Field (mathematics)","level":2,"score":0.567987859249115},{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.42774778604507446},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.42185789346694946},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.39803895354270935},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3198421001434326},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.25978147983551025},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3556536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3556536","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3556536","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},{"id":"pmh:oai:diglib.uibk.ac.at/:8648069","is_oa":true,"landing_page_url":"https://diglib.uibk.ac.at/doi/10.1145/3556536","pdf_url":"https://diglib.uibk.ac.at/ulbtirolfodok/content/titleinfo/8648069/full.pdf","source":{"id":"https://openalex.org/S4306401999","display_name":"Digital Library of the University of Innsbruck (University of Innsbruck)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I143397708","host_organization_name":"Innsbruck Medical University","host_organization_lineage":["https://openalex.org/I143397708"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Text"},{"id":"pmh:oai:dspace.library.uu.nl:1874/429603","is_oa":true,"landing_page_url":"https://dspace.library.uu.nl/handle/1874/429603","pdf_url":null,"source":{"id":"https://openalex.org/S4306401649","display_name":"Utrecht University Repository (Utrecht University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I193662353","host_organization_name":"Utrecht University","host_organization_lineage":["https://openalex.org/I193662353"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"}],"best_oa_location":{"id":"doi:10.1145/3556536","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3556536","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3556536","source":{"id":"https://openalex.org/S157921468","display_name":"ACM Computing Surveys","issn_l":"0360-0300","issn":["0360-0300","1557-7341"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Computing Surveys","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1837357248","display_name":null,"funder_award_id":"P33526","funder_id":"https://openalex.org/F4320321181","funder_display_name":"Austrian Science Fund"}],"funders":[{"id":"https://openalex.org/F4320321181","display_name":"Austrian Science Fund","ror":"https://ror.org/013tf3c58"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4292092796.pdf","grobid_xml":"https://content.openalex.org/works/W4292092796.grobid-xml"},"referenced_works_count":216,"referenced_works":["https://openalex.org/W51663010","https://openalex.org/W58619017","https://openalex.org/W91336075","https://openalex.org/W281665770","https://openalex.org/W395933519","https://openalex.org/W563405312","https://openalex.org/W582140561","https://openalex.org/W652470711","https://openalex.org/W786102430","https://openalex.org/W1015675232","https://openalex.org/W1412447802","https://openalex.org/W1500664688","https://openalex.org/W1511558225","https://openalex.org/W1530276735","https://openalex.org/W1547244403","https://openalex.org/W1554394657","https://openalex.org/W1587374567","https://openalex.org/W1607913544","https://openalex.org/W1757045719","https://openalex.org/W1774663064","https://openalex.org/W1835900096","https://openalex.org/W1965355809","https://openalex.org/W1967787801","https://openalex.org/W1969245231","https://openalex.org/W1971040550","https://openalex.org/W1973892538","https://openalex.org/W1977027027","https://openalex.org/W1979280215","https://openalex.org/W1981018178","https://openalex.org/W1982803242","https://openalex.org/W1985476803","https://openalex.org/W1990534247","https://openalex.org/W1992391624","https://openalex.org/W1999031685","https://openalex.org/W2007636657","https://openalex.org/W2009718036","https://openalex.org/W2011700584","https://openalex.org/W2016883491","https://openalex.org/W2019313057","https://openalex.org/W2020098476","https://openalex.org/W2020731303","https://openalex.org/W2023954349","https://openalex.org/W2030484290","https://openalex.org/W2034701578","https://openalex.org/W2040367556","https://openalex.org/W2042281163","https://openalex.org/W2044288351","https://openalex.org/W2045745608","https://openalex.org/W2046974451","https://openalex.org/W2050113838","https://openalex.org/W2054141820","https://openalex.org/W2057212716","https://openalex.org/W2058990114","https://openalex.org/W2060322584","https://openalex.org/W2062959588","https://openalex.org/W2067302566","https://openalex.org/W2069870183","https://openalex.org/W2076456228","https://openalex.org/W2078488685","https://openalex.org/W2078652719","https://openalex.org/W2088621849","https://openalex.org/W2099700594","https://openalex.org/W2100235918","https://openalex.org/W2100960835","https://openalex.org/W2101019781","https://openalex.org/W2101409192","https://openalex.org/W2104391728","https://openalex.org/W2104942079","https://openalex.org/W2105953200","https://openalex.org/W2107181104","https://openalex.org/W2110228583","https://openalex.org/W2111094216","https://openalex.org/W2112136322","https://openalex.org/W2112285779","https://openalex.org/W2112420033","https://openalex.org/W2112508839","https://openalex.org/W2112856797","https://openalex.org/W2112865076","https://openalex.org/W2114538223","https://openalex.org/W2116354394","https://openalex.org/W2118934678","https://openalex.org/W2119825970","https://openalex.org/W2123263696","https://openalex.org/W2124074612","https://openalex.org/W2124591829","https://openalex.org/W2125330369","https://openalex.org/W2134584261","https://openalex.org/W2136922540","https://openalex.org/W2138216384","https://openalex.org/W2144807535","https://openalex.org/W2145360759","https://openalex.org/W2148943147","https://openalex.org/W2153274408","https://openalex.org/W2155912844","https://openalex.org/W2161171125","https://openalex.org/W2161351857","https://openalex.org/W2161676175","https://openalex.org/W2162111811","https://openalex.org/W2163162311","https://openalex.org/W2171960770","https://openalex.org/W2191657943","https://openalex.org/W2207136466","https://openalex.org/W2210543184","https://openalex.org/W2219888463","https://openalex.org/W2221682297","https://openalex.org/W2223495811","https://openalex.org/W2238216200","https://openalex.org/W2338447355","https://openalex.org/W2340079414","https://openalex.org/W2341865734","https://openalex.org/W2346275821","https://openalex.org/W2349436533","https://openalex.org/W2414283705","https://openalex.org/W2418053903","https://openalex.org/W2474439552","https://openalex.org/W2492959872","https://openalex.org/W2497121487","https://openalex.org/W2511414107","https://openalex.org/W2516364559","https://openalex.org/W2533801348","https://openalex.org/W2539163507","https://openalex.org/W2560324319","https://openalex.org/W2562236173","https://openalex.org/W2575006718","https://openalex.org/W2586601519","https://openalex.org/W2588833253","https://openalex.org/W2591957553","https://openalex.org/W2605350416","https://openalex.org/W2612327332","https://openalex.org/W2625746539","https://openalex.org/W2732454496","https://openalex.org/W2766876172","https://openalex.org/W2767807341","https://openalex.org/W2772063975","https://openalex.org/W2784068709","https://openalex.org/W2787933113","https://openalex.org/W2787991113","https://openalex.org/W2792121711","https://openalex.org/W2793768763","https://openalex.org/W2795582157","https://openalex.org/W2797097514","https://openalex.org/W2807021761","https://openalex.org/W2829979434","https://openalex.org/W2886209086","https://openalex.org/W2892838935","https://openalex.org/W2893370267","https://openalex.org/W2894409023","https://openalex.org/W2897154134","https://openalex.org/W2906896588","https://openalex.org/W2912638141","https://openalex.org/W2929402962","https://openalex.org/W2943739336","https://openalex.org/W2946874020","https://openalex.org/W2951280825","https://openalex.org/W2955421345","https://openalex.org/W2957191877","https://openalex.org/W2963085847","https://openalex.org/W2963189767","https://openalex.org/W2969886764","https://openalex.org/W2972941122","https://openalex.org/W2973078969","https://openalex.org/W2987577970","https://openalex.org/W2998206837","https://openalex.org/W3011685549","https://openalex.org/W3014901735","https://openalex.org/W3023202929","https://openalex.org/W3030941844","https://openalex.org/W3040911419","https://openalex.org/W3045200674","https://openalex.org/W3047010778","https://openalex.org/W3048202504","https://openalex.org/W3094600186","https://openalex.org/W3098231197","https://openalex.org/W3099420497","https://openalex.org/W3099751307","https://openalex.org/W3099814932","https://openalex.org/W3100848837","https://openalex.org/W3101793783","https://openalex.org/W3102518922","https://openalex.org/W3104030692","https://openalex.org/W3105114834","https://openalex.org/W3105702364","https://openalex.org/W3106194691","https://openalex.org/W3120501274","https://openalex.org/W3122085188","https://openalex.org/W3122725723","https://openalex.org/W3125645198","https://openalex.org/W3128249766","https://openalex.org/W3128736136","https://openalex.org/W3134063549","https://openalex.org/W3135583098","https://openalex.org/W3137029514","https://openalex.org/W3154190849","https://openalex.org/W3158462925","https://openalex.org/W3193966380","https://openalex.org/W3195694877","https://openalex.org/W3197508101","https://openalex.org/W3199856156","https://openalex.org/W3201258060","https://openalex.org/W3201592279","https://openalex.org/W3205774243","https://openalex.org/W4220654158","https://openalex.org/W4226006895","https://openalex.org/W4226210383","https://openalex.org/W4226322861","https://openalex.org/W4226380897","https://openalex.org/W4238316224","https://openalex.org/W4254444566","https://openalex.org/W4286447870","https://openalex.org/W4288083766","https://openalex.org/W4288617757","https://openalex.org/W4297614008","https://openalex.org/W4297804276","https://openalex.org/W4300482433","https://openalex.org/W4300546939","https://openalex.org/W6781566418"],"related_works":["https://openalex.org/W2165912799","https://openalex.org/W2735662278","https://openalex.org/W2382615723","https://openalex.org/W4311804456","https://openalex.org/W1987484445","https://openalex.org/W2623658258","https://openalex.org/W2143413548","https://openalex.org/W1969219540","https://openalex.org/W2370459448","https://openalex.org/W2105067402"],"abstract_inverted_index":{"The":[0,115],"comprehensive":[1,99,149],"evaluation":[2,26,42,48,89,100,125,150],"of":[3,6,37,91,101,147,151],"the":[4,34,38,67,78,88,98,113,134,139,145],"performance":[5],"a":[7,11,102,119,148,170],"recommender":[8,62,81,92,103,152],"system":[9,104],"is":[10],"complex":[12],"endeavor:":[13],"many":[14],"facets":[15,29,109],"need":[16,161],"to":[17,122,136,162,166],"be":[18],"considered":[19],"in":[20,112,138],"configuring":[21],"an":[22,41,156],"adequate":[23,124],"and":[24,46,55,110,132,143,154,164],"effective":[25],"setting.":[27],"Such":[28],"include,":[30],"for":[31,69],"instance,":[32],"defining":[33],"specific":[35],"goals":[36],"evaluation,":[39],"choosing":[40],"method,":[43],"underlying":[44],"data,":[45],"suitable":[47],"metrics.":[49],"In":[50,84],"this":[51,58,129],"article,":[52],"we":[53,75,86,160],"consolidate":[54],"systematically":[56],"organize":[57],"dispersed":[59],"knowledge":[60],"on":[61,80,158],"systems":[63,72,82,93,153],"evaluation.":[64,83,94,114],"We":[65,95,141],"introduce":[66],"Framework":[68],"Evaluating":[70],"Recommender":[71],"(FEVR),":[73],"which":[74],"derive":[76],"from":[77],"discourse":[79],"FEVR,":[85],"categorize":[87],"space":[90],"postulate":[96],"that":[97,127],"frequently":[105],"requires":[106],"considering":[107],"multiple":[108],"perspectives":[111],"FEVR":[116],"framework":[117],"provides":[118,133],"structured":[120],"foundation":[121],"adopt":[123],"configurations":[126],"encompass":[128],"required":[130],"multi-facetedness":[131],"basis":[135],"advance":[137],"field.":[140],"outline":[142],"discuss":[144],"challenges":[146],"provide":[155],"outlook":[157],"what":[159],"embrace":[163],"do":[165],"move":[167],"forward":[168],"as":[169],"research":[171],"community.":[172]},"counts_by_year":[{"year":2026,"cited_by_count":20},{"year":2025,"cited_by_count":91},{"year":2024,"cited_by_count":73},{"year":2023,"cited_by_count":44},{"year":2022,"cited_by_count":1}],"updated_date":"2026-04-04T08:04:53.788161","created_date":"2025-10-10T00:00:00"}