{"id":"https://openalex.org/W2739622022","doi":"https://doi.org/10.1109/tlt.2017.2732349","title":"User-Centric Evaluation of Recommender Systems in Social Learning Platforms: Accuracy is Just the Tip of the Iceberg","display_name":"User-Centric Evaluation of Recommender Systems in Social Learning Platforms: Accuracy is Just the Tip of the Iceberg","publication_year":2017,"publication_date":"2017-07-27","ids":{"openalex":"https://openalex.org/W2739622022","doi":"https://doi.org/10.1109/tlt.2017.2732349","mag":"2739622022"},"language":"en","primary_location":{"id":"doi:10.1109/tlt.2017.2732349","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tlt.2017.2732349","pdf_url":null,"source":{"id":"https://openalex.org/S130363450","display_name":"IEEE Transactions on Learning Technologies","issn_l":"1939-1382","issn":["1939-1382","2372-0050"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Learning Technologies","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://research.ou.nl/en/publications/bea2bf9c-5c94-4586-aa0a-1a10ec2e50c8","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073613176","display_name":"Soude Fazeli","orcid":"https://orcid.org/0000-0003-1250-994X"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Soude Fazeli","raw_affiliation_strings":["Technische Universiteit Delft, Delft, Zuid-Holland, NL"],"raw_orcid":"https://orcid.org/0000-0003-1250-994X","affiliations":[{"raw_affiliation_string":"Technische Universiteit Delft, Delft, Zuid-Holland, NL","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027722484","display_name":"Hendrik Drachsler","orcid":"https://orcid.org/0000-0001-8407-5314"},"institutions":[{"id":"https://openalex.org/I114090438","display_name":"Goethe University Frankfurt","ror":"https://ror.org/04cvxnb49","country_code":"DE","type":"education","lineage":["https://openalex.org/I114090438"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hendrik Drachsler","raw_affiliation_strings":["Goethe University Frankfurt, the German Institute of International Educational Research (DIPF), and the Open University of the Netherlands, Heerlen 6419 AT, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Goethe University Frankfurt, the German Institute of International Educational Research (DIPF), and the Open University of the Netherlands, Heerlen 6419 AT, The Netherlands","institution_ids":["https://openalex.org/I114090438"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110231201","display_name":"Marlies Bitter-Rijpkema","orcid":null},"institutions":[{"id":"https://openalex.org/I7876267","display_name":"Open University of the Netherlands","ror":"https://ror.org/018dfmf50","country_code":"NL","type":"education","lineage":["https://openalex.org/I7876267"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Marlies Bitter-Rijpkema","raw_affiliation_strings":["Open Universiteit, Heerlen, Limburg, NL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open Universiteit, Heerlen, Limburg, NL","institution_ids":["https://openalex.org/I7876267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088658885","display_name":"Francis Brouns","orcid":"https://orcid.org/0000-0002-6240-2684"},"institutions":[{"id":"https://openalex.org/I7876267","display_name":"Open University of the Netherlands","ror":"https://ror.org/018dfmf50","country_code":"NL","type":"education","lineage":["https://openalex.org/I7876267"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Francis Brouns","raw_affiliation_strings":["Open Universiteit, Heerlen, Limburg, NL"],"raw_orcid":"https://orcid.org/0000-0002-6240-2684","affiliations":[{"raw_affiliation_string":"Open Universiteit, Heerlen, Limburg, NL","institution_ids":["https://openalex.org/I7876267"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082609435","display_name":"Wim van der Vegt","orcid":"https://orcid.org/0000-0002-7732-1561"},"institutions":[{"id":"https://openalex.org/I7876267","display_name":"Open University of the Netherlands","ror":"https://ror.org/018dfmf50","country_code":"NL","type":"education","lineage":["https://openalex.org/I7876267"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Wim van der Vegt Brouns","raw_affiliation_strings":["Open Universiteit, Heerlen, Limburg, NL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open Universiteit, Heerlen, Limburg, NL","institution_ids":["https://openalex.org/I7876267"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031402391","display_name":"Peter Sloep","orcid":"https://orcid.org/0000-0003-4009-4980"},"institutions":[{"id":"https://openalex.org/I7876267","display_name":"Open University of the Netherlands","ror":"https://ror.org/018dfmf50","country_code":"NL","type":"education","lineage":["https://openalex.org/I7876267"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Peter B. Sloep","raw_affiliation_strings":["Open Universiteit, Heerlen, Limburg, NL"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Open Universiteit, Heerlen, Limburg, NL","institution_ids":["https://openalex.org/I7876267"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5073613176"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":5.5648,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":{"value":0.96206618,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"11","issue":"3","first_page":"294","last_page":"306"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9908000230789185,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11478","display_name":"Caching and Content Delivery","score":0.9717000126838684,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/recommender-system","display_name":"Recommender system","score":0.9330686330795288},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8821696043014526},{"id":"https://openalex.org/keywords/serendipity","display_name":"Serendipity","score":0.7152169942855835},{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.5892977118492126},{"id":"https://openalex.org/keywords/user-experience-design","display_name":"User experience design","score":0.4262147843837738},{"id":"https://openalex.org/keywords/user-satisfaction","display_name":"User satisfaction","score":0.4143417179584503},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3716805577278137},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.33829957246780396},{"id":"https://openalex.org/keywords/human\u2013computer-interaction","display_name":"Human\u2013computer interaction","score":0.2535681128501892}],"concepts":[{"id":"https://openalex.org/C557471498","wikidata":"https://www.wikidata.org/wiki/Q554950","display_name":"Recommender system","level":2,"score":0.9330686330795288},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8821696043014526},{"id":"https://openalex.org/C2779119418","wikidata":"https://www.wikidata.org/wiki/Q166039","display_name":"Serendipity","level":2,"score":0.7152169942855835},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.5892977118492126},{"id":"https://openalex.org/C201025465","wikidata":"https://www.wikidata.org/wiki/Q11248500","display_name":"User experience design","level":2,"score":0.4262147843837738},{"id":"https://openalex.org/C3017893058","wikidata":"https://www.wikidata.org/wiki/Q999278","display_name":"User satisfaction","level":2,"score":0.4143417179584503},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3716805577278137},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.33829957246780396},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.2535681128501892},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C27206212","wikidata":"https://www.wikidata.org/wiki/Q34178","display_name":"Theology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/tlt.2017.2732349","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tlt.2017.2732349","pdf_url":null,"source":{"id":"https://openalex.org/S130363450","display_name":"IEEE Transactions on Learning Technologies","issn_l":"1939-1382","issn":["1939-1382","2372-0050"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Learning Technologies","raw_type":"journal-article"},{"id":"pmh:oai:dspace.ou.nl:1820/8650","is_oa":false,"landing_page_url":"http://hdl.handle.net/1820/8650","pdf_url":null,"source":{"id":"https://openalex.org/S4306401002","display_name":"DSpace (Open University in the Netherlands)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I7876267","host_organization_name":"Open University of the Netherlands","host_organization_lineage":["https://openalex.org/I7876267"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Article"},{"id":"pmh:ou:oai:research.ou.nl:publications/bea2bf9c-5c94-4586-aa0a-1a10ec2e50c8","is_oa":true,"landing_page_url":"https://research.ou.nl/en/publications/bea2bf9c-5c94-4586-aa0a-1a10ec2e50c8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Learning Technologies, 11(3), 294 - 306. IEEE","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:ou:oai:research.ou.nl:publications/bea2bf9c-5c94-4586-aa0a-1a10ec2e50c8","is_oa":true,"landing_page_url":"https://research.ou.nl/en/publications/bea2bf9c-5c94-4586-aa0a-1a10ec2e50c8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Transactions on Learning Technologies, 11(3), 294 - 306. IEEE","raw_type":"info:eu-repo/semantics/article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320300","display_name":"European Commission","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":48,"referenced_works":["https://openalex.org/W129119727","https://openalex.org/W1452699976","https://openalex.org/W1507584106","https://openalex.org/W1552748051","https://openalex.org/W1583059097","https://openalex.org/W1598687747","https://openalex.org/W1690919088","https://openalex.org/W1939407039","https://openalex.org/W1971040550","https://openalex.org/W1988139567","https://openalex.org/W2007636657","https://openalex.org/W2018525987","https://openalex.org/W2049670925","https://openalex.org/W2058738785","https://openalex.org/W2090883204","https://openalex.org/W2101019781","https://openalex.org/W2102384900","https://openalex.org/W2102456297","https://openalex.org/W2107584556","https://openalex.org/W2114538223","https://openalex.org/W2119825970","https://openalex.org/W2125330369","https://openalex.org/W2128438887","https://openalex.org/W2133266261","https://openalex.org/W2134584261","https://openalex.org/W2135943618","https://openalex.org/W2140310134","https://openalex.org/W2145360759","https://openalex.org/W2150494057","https://openalex.org/W2150978351","https://openalex.org/W2153111836","https://openalex.org/W2153580689","https://openalex.org/W2162111811","https://openalex.org/W2167829442","https://openalex.org/W2171109587","https://openalex.org/W2199058847","https://openalex.org/W2223495811","https://openalex.org/W2531935925","https://openalex.org/W4235114045","https://openalex.org/W4242721106","https://openalex.org/W4285719527","https://openalex.org/W6630343348","https://openalex.org/W6640693739","https://openalex.org/W6680830989","https://openalex.org/W6684328791","https://openalex.org/W6728594466","https://openalex.org/W7018424506","https://openalex.org/W7030472626"],"related_works":["https://openalex.org/W4205615116","https://openalex.org/W2154428221","https://openalex.org/W2531086667","https://openalex.org/W2045676999","https://openalex.org/W35325473","https://openalex.org/W1876349746","https://openalex.org/W4312541699","https://openalex.org/W2901850002","https://openalex.org/W4226304833","https://openalex.org/W2914310705"],"abstract_inverted_index":{"Recommender":[0],"systems":[1,13,81,159,174],"provide":[2],"users":[3],"with":[4,147],"content":[5],"they":[6],"might":[7],"be":[8,44],"interested":[9],"in.":[10],"Conventionally,":[11],"recommender":[12,29,49,80,123,158,173],"are":[14,58,142],"evaluated":[15],"mostly":[16],"by":[17],"using":[18],"prediction":[19,163],"accuracy":[20,133,164],"metrics":[21,197],"only.":[22],"But,":[23],"the":[24,64,89,106,111,154,176,180,194],"ultimate":[25],"goal":[26],"of":[27,78,93,114,121,132,157,162,172,193],"a":[28,48,52,75,85,102,122],"system":[30,50,124],"is":[31,125],"to":[32,63,109,127,169],"increase":[33],"user":[34,40,119,177],"satisfaction.":[35],"Therefore,":[36],"evaluations":[37,57,116],"that":[38,138,153],"measure":[39],"satisfaction":[41,120],"should":[42],"also":[43,100],"performed":[45],"before":[46],"deploying":[47],"in":[51,88,130,145,160,186],"real":[53],"target":[54],"environment.":[55],"Such":[56],"laborious":[59],"and":[60,117,204],"complicated":[61],"compared":[62],"traditional,":[65],"data-centric":[66,103,148],"evaluations,":[67],"though.":[68],"In":[69],"this":[70],"study,":[71],"we":[72],"carried":[73],"out":[74],"user-centric":[76,115,139,181],"evaluation":[77,104,140,149,156,182],"state-of-the-art":[79],"as":[82,84],"well":[83],"graph-based":[86],"approach":[87],"ecologically":[90],"valid":[91],"setting":[92],"an":[94],"authentic":[95],"social":[96],"learning":[97],"platform.":[98],"We":[99,151],"conducted":[101],"on":[105,175,191],"same":[107],"data":[108],"investigate":[110],"added":[112],"value":[113],"how":[118,187],"related":[126],"its":[128],"performance":[129,171],"terms":[131,161],"metrics.":[134],"Our":[135],"findings":[136],"suggest":[137],"results":[141],"not":[143,167],"necessarily":[144],"line":[146],"results.":[150],"conclude":[152],"traditional":[155],"only":[165],"does":[166],"suffice":[168],"judge":[170],"side.":[178],"Moreover,":[179],"provides":[183],"valuable":[184],"insights":[185],"candidate":[188],"algorithms":[189],"perform":[190],"each":[192],"five":[195],"quality":[196],"for":[198],"recommendations:":[199],"usefulness,":[200],"accuracy,":[201],"novelty,":[202],"diversity,":[203],"serendipity.":[205]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":8},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":8},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":2}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}