{"id":"https://openalex.org/W2774321216","doi":"https://doi.org/10.1007/s13042-017-0762-9","title":"How good your recommender system is? A survey on evaluations in recommendation","display_name":"How good your recommender system is? A survey on evaluations in recommendation","publication_year":2017,"publication_date":"2017-12-14","ids":{"openalex":"https://openalex.org/W2774321216","doi":"https://doi.org/10.1007/s13042-017-0762-9","mag":"2774321216"},"language":"en","primary_location":{"id":"doi:10.1007/s13042-017-0762-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13042-017-0762-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs13042-017-0762-9.pdf","source":{"id":"https://openalex.org/S2764999920","display_name":"International Journal of Machine Learning and Cybernetics","issn_l":"1868-8071","issn":["1868-8071","1868-808X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Machine Learning and Cybernetics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007%2Fs13042-017-0762-9.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101634252","display_name":"Thiago Silveira","orcid":"https://orcid.org/0000-0002-6854-1108"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Thiago Silveira","raw_affiliation_strings":["Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100402996","display_name":"Min Zhang","orcid":"https://orcid.org/0000-0003-3158-1920"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Min Zhang","raw_affiliation_strings":["Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005130812","display_name":"Lin Xiao","orcid":"https://orcid.org/0000-0003-3172-3490"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Lin","raw_affiliation_strings":["Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100668121","display_name":"Yiqun Liu","orcid":"https://orcid.org/0000-0002-0140-4512"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiqun Liu","raw_affiliation_strings":["Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100760812","display_name":"Shaoping Ma","orcid":"https://orcid.org/0000-0002-8762-8268"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shaoping Ma","raw_affiliation_strings":["Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Tsinghua National Laboratory for Information Science and Technology, Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5101634252"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":{"value":2790,"currency":"EUR","value_usd":3590},"apc_paid":{"value":2790,"currency":"EUR","value_usd":3590},"fwci":20.2343,"has_fulltext":true,"cited_by_count":259,"citation_normalized_percentile":{"value":0.99304844,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":"10","issue":"5","first_page":"813","last_page":"831"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10824","display_name":"Image Retrieval and Classification Techniques","score":0.9916999936103821,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9702000021934509,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.9040666222572327},{"id":"https://openalex.org/keywords/computational-intelligence","display_name":"Computational intelligence","score":0.7007240056991577},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6531521677970886},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3718770146369934},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3402032256126404}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.9040666222572327},{"id":"https://openalex.org/C139502532","wikidata":"https://www.wikidata.org/wiki/Q1122090","display_name":"Computational intelligence","level":2,"score":0.7007240056991577},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6531521677970886},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3718770146369934},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3402032256126404}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s13042-017-0762-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13042-017-0762-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs13042-017-0762-9.pdf","source":{"id":"https://openalex.org/S2764999920","display_name":"International Journal of Machine Learning and Cybernetics","issn_l":"1868-8071","issn":["1868-8071","1868-808X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Machine Learning and Cybernetics","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s13042-017-0762-9","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s13042-017-0762-9","pdf_url":"https://link.springer.com/content/pdf/10.1007%2Fs13042-017-0762-9.pdf","source":{"id":"https://openalex.org/S2764999920","display_name":"International Journal of Machine Learning and Cybernetics","issn_l":"1868-8071","issn":["1868-8071","1868-808X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"International Journal of Machine Learning and Cybernetics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3892115987","display_name":null,"funder_award_id":"61532011","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8096314617","display_name":null,"funder_award_id":"61672311","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2774321216.pdf","grobid_xml":"https://content.openalex.org/works/W2774321216.grobid-xml"},"referenced_works_count":40,"referenced_works":["https://openalex.org/W824612272","https://openalex.org/W1547244403","https://openalex.org/W1560799855","https://openalex.org/W1568657496","https://openalex.org/W1690919088","https://openalex.org/W1967787801","https://openalex.org/W1971040550","https://openalex.org/W2003810236","https://openalex.org/W2009718036","https://openalex.org/W2011700584","https://openalex.org/W2012903477","https://openalex.org/W2014358499","https://openalex.org/W2025605741","https://openalex.org/W2031768829","https://openalex.org/W2033213271","https://openalex.org/W2045453095","https://openalex.org/W2049670925","https://openalex.org/W2056564184","https://openalex.org/W2086579517","https://openalex.org/W2108630796","https://openalex.org/W2110703683","https://openalex.org/W2111094216","https://openalex.org/W2133029037","https://openalex.org/W2144807535","https://openalex.org/W2146682077","https://openalex.org/W2155912844","https://openalex.org/W2161676175","https://openalex.org/W2164646033","https://openalex.org/W2171960770","https://openalex.org/W2185318434","https://openalex.org/W2210543184","https://openalex.org/W2373122082","https://openalex.org/W2514091133","https://openalex.org/W2518353250","https://openalex.org/W2561618650","https://openalex.org/W2620166481","https://openalex.org/W2726080841","https://openalex.org/W2875879452","https://openalex.org/W4205171160","https://openalex.org/W4243863304"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W4390273403","https://openalex.org/W4386781444","https://openalex.org/W2150182025","https://openalex.org/W3092950680","https://openalex.org/W3197542405","https://openalex.org/W2056712470","https://openalex.org/W3125580266"],"abstract_inverted_index":{"Recommender":[0],"Systems":[1],"have":[2,15,64,84,97,112],"become":[3],"a":[4,9,50,122],"very":[5],"useful":[6],"tool":[7],"for":[8,76],"large":[10],"variety":[11],"of":[12,33,56,62,121,127],"domains.":[13],"Researchers":[14],"been":[16,65,98,113],"attempting":[17],"to":[18,24,28,41,43,103,153,170],"improve":[19],"their":[20,171],"algorithms":[21],"in":[22,37,115],"order":[23],"issue":[25],"better":[26],"predictions":[27,47],"the":[29,34,38,46,54,101,105,119,140,162,165],"users.":[30],"However,":[31],"one":[32],"current":[35],"challenges":[36],"area":[39],"refers":[40],"how":[42],"properly":[44],"evaluate":[45,154],"generated":[48],"by":[49],"recommender":[51],"system.":[52],"In":[53,79,156],"extent":[55],"offline":[57],"evaluations,":[58],"some":[59,86],"traditional":[60,130],"concepts":[61,88,96,147],"evaluation":[63],"explored,":[66],"such":[67,89],"as":[68,90],"accuracy,":[69],"Root":[70],"Mean":[71],"Square":[72],"Error":[73],"and":[74,93,110,132,138,148,173],"P@N":[75],"top-k":[77],"recommendations.":[78,155],"recent":[80,133],"years,":[81],"more":[82],"research":[83,142],"proposed":[85,114],"new":[87],"novelty,":[91],"diversity":[92],"serendipity.":[94],"These":[95],"addressed":[99],"with":[100],"goal":[102],"satisfy":[104],"users\u2019":[106],"requirements.":[107],"Numerous":[108],"definitions":[109,145],"metrics":[111,131,150],"previous":[116],"work.":[117],"On":[118],"absence":[120],"specific":[123],"summarization":[124],"on":[125,178],"evaluations":[126],"recommendation":[128],"combining":[129],"progresses,":[134],"this":[135,158],"paper":[136],"surveys":[137],"organizes":[139],"main":[141],"that":[143],"present":[144],"about":[146],"propose":[149],"or":[151],"strategies":[152],"addition,":[157],"survey":[159],"also":[160],"settles":[161],"relationship":[163],"between":[164],"concepts,":[166],"categorizes":[167],"them":[168],"according":[169],"objectives":[172],"suggests":[174],"potential":[175],"future":[176],"topics":[177],"user":[179],"satisfaction.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":13},{"year":2025,"cited_by_count":40},{"year":2024,"cited_by_count":41},{"year":2023,"cited_by_count":46},{"year":2022,"cited_by_count":33},{"year":2021,"cited_by_count":46},{"year":2020,"cited_by_count":28},{"year":2019,"cited_by_count":8},{"year":2018,"cited_by_count":4}],"updated_date":"2026-04-14T08:04:32.555800","created_date":"2025-10-10T00:00:00"}