{"id":"https://openalex.org/W2153274408","doi":"https://doi.org/10.1145/2532508.2532512","title":"Research paper recommender system evaluation","display_name":"Research paper recommender system evaluation","publication_year":2013,"publication_date":"2013-10-12","ids":{"openalex":"https://openalex.org/W2153274408","doi":"https://doi.org/10.1145/2532508.2532512","mag":"2153274408"},"language":"en","primary_location":{"id":"doi:10.1145/2532508.2532512","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2532508.2532512","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Workshop on Reproducibility and Replication in Recommender Systems Evaluation","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081348430","display_name":"Joeran Beel","orcid":"https://orcid.org/0000-0002-4537-5573"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Joeran Beel","raw_affiliation_strings":["Docear, Magdeburg, Germany"],"affiliations":[{"raw_affiliation_string":"Docear, Magdeburg, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103684976","display_name":"Stefan Langer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Stefan Langer","raw_affiliation_strings":["Docear, Magdeburg, Germany"],"affiliations":[{"raw_affiliation_string":"Docear, Magdeburg, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079625684","display_name":"Marcel Genzmehr","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Marcel Genzmehr","raw_affiliation_strings":["Docear, Magdeburg, Germany"],"affiliations":[{"raw_affiliation_string":"Docear, Magdeburg, Germany","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058837356","display_name":"B\u00e9la Gipp","orcid":"https://orcid.org/0000-0001-6522-3019"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bela Gipp","raw_affiliation_strings":["Univ. of California, Berkeley"],"affiliations":[{"raw_affiliation_string":"Univ. of California, Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013617539","display_name":"Corinna Breitinger","orcid":"https://orcid.org/0000-0001-6586-0392"},"institutions":[{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Corinna Breitinger","raw_affiliation_strings":["Univ. of California, Berkeley"],"affiliations":[{"raw_affiliation_string":"Univ. of California, Berkeley","institution_ids":["https://openalex.org/I95457486"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075663234","display_name":"Andreas N\u00fcrnberger","orcid":"https://orcid.org/0000-0003-4311-0624"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andreas N\u00fcrnberger","raw_affiliation_strings":["OvGU, FIN, ITI, Magdeburg, Germany"],"affiliations":[{"raw_affiliation_string":"OvGU, FIN, ITI, Magdeburg, Germany","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5081348430"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":27.081,"has_fulltext":true,"cited_by_count":158,"citation_normalized_percentile":{"value":0.99450175,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"15","last_page":"22"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13274","display_name":"Expert finding and Q&A systems","score":0.9943000078201294,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.7682918906211853},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7589402198791504},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.7126336693763733},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.5494918823242188},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.4580196142196655},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.38828614354133606},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1869189441204071}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.7682918906211853},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7589402198791504},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.7126336693763733},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.5494918823242188},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.4580196142196655},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.38828614354133606},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1869189441204071},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/2532508.2532512","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2532508.2532512","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the International Workshop on Reproducibility and Replication in Recommender Systems Evaluation","raw_type":"proceedings-article"},{"id":"pmh:oai:kops.uni-konstanz.de:123456789/30897","is_oa":false,"landing_page_url":"http://nbn-resolving.de/urn:nbn:de:bsz:352-0-285593","pdf_url":null,"source":{"id":"https://openalex.org/S4306401487","display_name":"KOPS (University of Konstanz)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I189712700","host_organization_name":"University of Konstanz","host_organization_lineage":["https://openalex.org/I189712700"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"ALEJANDRO BELLOGIN, ,  ed.. Proceedings of the International Workshop on Reproducibility and Replication in Recommender Systems Evaluation. New York: ACM, 2013, pp. 15-22. ISBN 978-1-4503-2465-6. Available under: doi: 10.1145/2532508.2532512","raw_type":"doc-type:Text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":115,"referenced_works":["https://openalex.org/W29625609","https://openalex.org/W43235638","https://openalex.org/W80379025","https://openalex.org/W115023223","https://openalex.org/W127652380","https://openalex.org/W142212369","https://openalex.org/W150278719","https://openalex.org/W1483715664","https://openalex.org/W1490345502","https://openalex.org/W1493430969","https://openalex.org/W1495331069","https://openalex.org/W1496262514","https://openalex.org/W1510478425","https://openalex.org/W1511198016","https://openalex.org/W1532325895","https://openalex.org/W1532470117","https://openalex.org/W1532474497","https://openalex.org/W1532852017","https://openalex.org/W1545505557","https://openalex.org/W1560536102","https://openalex.org/W1573069972","https://openalex.org/W1590650811","https://openalex.org/W1690919088","https://openalex.org/W1801697423","https://openalex.org/W1825004802","https://openalex.org/W1828964362","https://openalex.org/W1965675974","https://openalex.org/W1970572480","https://openalex.org/W1971040550","https://openalex.org/W1977543143","https://openalex.org/W1978059262","https://openalex.org/W1985476803","https://openalex.org/W1988877785","https://openalex.org/W1995326326","https://openalex.org/W1998718108","https://openalex.org/W2000374638","https://openalex.org/W2000613421","https://openalex.org/W2002642763","https://openalex.org/W2003578012","https://openalex.org/W2003790488","https://openalex.org/W2006987993","https://openalex.org/W2007628822","https://openalex.org/W2014962039","https://openalex.org/W2017591847","https://openalex.org/W2029249040","https://openalex.org/W2030278476","https://openalex.org/W2031456732","https://openalex.org/W2040974535","https://openalex.org/W2049217466","https://openalex.org/W2055899255","https://openalex.org/W2061172803","https://openalex.org/W2062340319","https://openalex.org/W2065260283","https://openalex.org/W2066343199","https://openalex.org/W2069342754","https://openalex.org/W2084177123","https://openalex.org/W2084665596","https://openalex.org/W2088772104","https://openalex.org/W2090715008","https://openalex.org/W2091389042","https://openalex.org/W2093945069","https://openalex.org/W2097166075","https://openalex.org/W2102378894","https://openalex.org/W2102870745","https://openalex.org/W2104343861","https://openalex.org/W2105638237","https://openalex.org/W2108630796","https://openalex.org/W2109992782","https://openalex.org/W2112293095","https://openalex.org/W2112589941","https://openalex.org/W2116655493","https://openalex.org/W2118934678","https://openalex.org/W2121516710","https://openalex.org/W2122778642","https://openalex.org/W2124470082","https://openalex.org/W2124610812","https://openalex.org/W2128629010","https://openalex.org/W2131531789","https://openalex.org/W2132895047","https://openalex.org/W2135389419","https://openalex.org/W2136849556","https://openalex.org/W2137208510","https://openalex.org/W2139375986","https://openalex.org/W2141763599","https://openalex.org/W2142574815","https://openalex.org/W2143334947","https://openalex.org/W2143844793","https://openalex.org/W2145677303","https://openalex.org/W2146936057","https://openalex.org/W2154976144","https://openalex.org/W2155576308","https://openalex.org/W2156186232","https://openalex.org/W2159384568","https://openalex.org/W2161676175","https://openalex.org/W2162338820","https://openalex.org/W2165611133","https://openalex.org/W2165636119","https://openalex.org/W2171603371","https://openalex.org/W2171960770","https://openalex.org/W2246213091","https://openalex.org/W2262256900","https://openalex.org/W2510089130","https://openalex.org/W2544305578","https://openalex.org/W2582765108","https://openalex.org/W2613258304","https://openalex.org/W2998206837","https://openalex.org/W3145623066","https://openalex.org/W4213009331","https://openalex.org/W4285719527","https://openalex.org/W6606005913","https://openalex.org/W6629726061","https://openalex.org/W6637295978","https://openalex.org/W6664578038","https://openalex.org/W6667616612","https://openalex.org/W6683624839"],"related_works":["https://openalex.org/W2383111961","https://openalex.org/W2365952365","https://openalex.org/W2352448290","https://openalex.org/W4390273403","https://openalex.org/W2380820513","https://openalex.org/W4386781444","https://openalex.org/W2913146933","https://openalex.org/W2372385138","https://openalex.org/W4296359239","https://openalex.org/W2101155126"],"abstract_inverted_index":{"Over":[0],"80":[1,129],"approaches":[2,10,31,42,110,130,135],"for":[3,111],"academic":[4,112],"literature":[5,113],"recommendation":[6,109],"exist":[7],"today.":[8],"The":[9],"were":[11,45,53],"introduced":[12],"and":[13,26,32,82,90],"evaluated":[14,50,55],"in":[15,95,123],"more":[16,127],"than":[17,128],"170":[18],"research":[19],"articles,":[20],"as":[21,23],"well":[22],"patents,":[24],"presentations":[25],"blogs.":[27],"We":[28],"reviewed":[29],"these":[30,89],"found":[33],"most":[34,116],"evaluations":[35],"to":[36,88,106],"contain":[37],"major":[38],"shortcomings.":[39],"Of":[40,59],"the":[41,49,60,75,115,124,132],"proposed,":[43],"21%":[44],"not":[46,54,72,104],"evaluated.":[47],"Among":[48],"approaches,":[51],"19%":[52],"against":[56],"a":[57],"baseline.":[58],"user":[61],"studies":[62],"performed,":[63],"60%":[64],"had":[65],"15":[66],"or":[67,70],"fewer":[68],"participants":[69],"did":[71],"report":[73],"on":[74,80],"number":[76],"of":[77,126],"participants.":[78],"Information":[79],"runtime":[81],"coverage":[83],"was":[84],"rarely":[85],"provided.":[86],"Due":[87],"several":[91],"other":[92],"shortcomings":[93],"described":[94],"this":[96],"paper,":[97],"we":[98],"conclude":[99],"that":[100],"it":[101],"is":[102,120],"currently":[103],"possible":[105],"determine":[107],"which":[108],"are":[114,136],"promising.":[117],"However,":[118],"there":[119],"little":[121],"value":[122],"existence":[125],"if":[131],"best":[133],"performing":[134],"unknown.":[137]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":32},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":18},{"year":2020,"cited_by_count":15},{"year":2019,"cited_by_count":13},{"year":2018,"cited_by_count":13},{"year":2017,"cited_by_count":9},{"year":2016,"cited_by_count":12},{"year":2015,"cited_by_count":12},{"year":2014,"cited_by_count":7},{"year":2013,"cited_by_count":1}],"updated_date":"2026-03-25T23:56:10.502304","created_date":"2016-06-24T00:00:00"}