{"id":"https://openalex.org/W4390396506","doi":"https://doi.org/10.3390/fi16010012","title":"Evaluating Embeddings from Pre-Trained Language Models and Knowledge Graphs for Educational Content Recommendation","display_name":"Evaluating Embeddings from Pre-Trained Language Models and Knowledge Graphs for Educational Content Recommendation","publication_year":2023,"publication_date":"2023-12-29","ids":{"openalex":"https://openalex.org/W4390396506","doi":"https://doi.org/10.3390/fi16010012"},"language":"en","primary_location":{"id":"doi:10.3390/fi16010012","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi16010012","pdf_url":"https://www.mdpi.com/1999-5903/16/1/12/pdf?version=1703820698","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1999-5903/16/1/12/pdf?version=1703820698","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004942213","display_name":"Xiu Li","orcid":"https://orcid.org/0000-0002-7860-1784"},"institutions":[{"id":"https://openalex.org/I161593684","display_name":"Stockholm University","ror":"https://ror.org/05f0yaq80","country_code":"SE","type":"education","lineage":["https://openalex.org/I161593684"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Xiu Li","raw_affiliation_strings":["Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-7860-1784","affiliations":[{"raw_affiliation_string":"Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden","institution_ids":["https://openalex.org/I161593684"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5022159718","display_name":"Aron Henriksson","orcid":null},"institutions":[{"id":"https://openalex.org/I161593684","display_name":"Stockholm University","ror":"https://ror.org/05f0yaq80","country_code":"SE","type":"education","lineage":["https://openalex.org/I161593684"]}],"countries":["SE"],"is_corresponding":true,"raw_author_name":"Aron Henriksson","raw_affiliation_strings":["Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden","institution_ids":["https://openalex.org/I161593684"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074751181","display_name":"Martin Duneld","orcid":null},"institutions":[{"id":"https://openalex.org/I161593684","display_name":"Stockholm University","ror":"https://ror.org/05f0yaq80","country_code":"SE","type":"education","lineage":["https://openalex.org/I161593684"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Martin Duneld","raw_affiliation_strings":["Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden","institution_ids":["https://openalex.org/I161593684"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017174955","display_name":"Jalal Nouri","orcid":null},"institutions":[{"id":"https://openalex.org/I161593684","display_name":"Stockholm University","ror":"https://ror.org/05f0yaq80","country_code":"SE","type":"education","lineage":["https://openalex.org/I161593684"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Jalal Nouri","raw_affiliation_strings":["Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden","institution_ids":["https://openalex.org/I161593684"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069516494","display_name":"Yongchao Wu","orcid":"https://orcid.org/0000-0002-0945-707X"},"institutions":[{"id":"https://openalex.org/I161593684","display_name":"Stockholm University","ror":"https://ror.org/05f0yaq80","country_code":"SE","type":"education","lineage":["https://openalex.org/I161593684"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Yongchao Wu","raw_affiliation_strings":["Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-0945-707X","affiliations":[{"raw_affiliation_string":"Department of Computer and Systems Sciences, Stockholm University, NOD-Huset, Borgarfjordsgatan 12, 16455 Stockholm, Sweden","institution_ids":["https://openalex.org/I161593684"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5022159718"],"corresponding_institution_ids":["https://openalex.org/I161593684"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":5.2543,"has_fulltext":true,"cited_by_count":32,"citation_normalized_percentile":{"value":0.96537336,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"16","issue":"1","first_page":"12","last_page":"12"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11273","display_name":"Advanced Graph Neural Networks","score":0.9940000176429749,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8572515249252319},{"id":"https://openalex.org/keywords/graph","display_name":"Graph","score":0.573448896408081},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5445116758346558},{"id":"https://openalex.org/keywords/cornerstone","display_name":"Cornerstone","score":0.539598822593689},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5257443189620972},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5143612623214722},{"id":"https://openalex.org/keywords/knowledge-graph","display_name":"Knowledge graph","score":0.4913589060306549},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.45237481594085693},{"id":"https://openalex.org/keywords/recall","display_name":"Recall","score":0.43482595682144165},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42964109778404236},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3224064111709595},{"id":"https://openalex.org/keywords/theoretical-computer-science","display_name":"Theoretical computer science","score":0.22443553805351257}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8572515249252319},{"id":"https://openalex.org/C132525143","wikidata":"https://www.wikidata.org/wiki/Q141488","display_name":"Graph","level":2,"score":0.573448896408081},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5445116758346558},{"id":"https://openalex.org/C2780616401","wikidata":"https://www.wikidata.org/wiki/Q1133673","display_name":"Cornerstone","level":2,"score":0.539598822593689},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5257443189620972},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5143612623214722},{"id":"https://openalex.org/C2987255567","wikidata":"https://www.wikidata.org/wiki/Q33002955","display_name":"Knowledge graph","level":2,"score":0.4913589060306549},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.45237481594085693},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.43482595682144165},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42964109778404236},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3224064111709595},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.22443553805351257},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.3390/fi16010012","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi16010012","pdf_url":"https://www.mdpi.com/1999-5903/16/1/12/pdf?version=1703820698","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},{"id":"pmh:oai:DiVA.org:su-224920","is_oa":true,"landing_page_url":"http://urn.kb.se/resolve?urn=urn:nbn:se:su:diva-224920","pdf_url":"https://su.diva-portal.org/smash/get/diva2:1823387/FULLTEXT01","source":{"id":"https://openalex.org/S4306401559","display_name":"KTH Publication Database DiVA (KTH Royal Institute of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:RePEc:gam:jftint:v:16:y:2023:i:1:p:12-:d:1309732","is_oa":false,"landing_page_url":"https://www.mdpi.com/1999-5903/16/1/12/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:doaj.org/article:59d6b01a55b2403aa4e1f3f545bdbe6e","is_oa":true,"landing_page_url":"https://doaj.org/article/59d6b01a55b2403aa4e1f3f545bdbe6e","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Future Internet, Vol 16, Iss 1, p 12 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.3390/fi16010012","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi16010012","pdf_url":"https://www.mdpi.com/1999-5903/16/1/12/pdf?version=1703820698","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.8799999952316284}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4390396506.pdf"},"referenced_works_count":52,"referenced_works":["https://openalex.org/W102708294","https://openalex.org/W1503259811","https://openalex.org/W1566018662","https://openalex.org/W2022166150","https://openalex.org/W2080133951","https://openalex.org/W2094728533","https://openalex.org/W2123142779","https://openalex.org/W2127795553","https://openalex.org/W2131744502","https://openalex.org/W2251295945","https://openalex.org/W2509893387","https://openalex.org/W2561529111","https://openalex.org/W2571811098","https://openalex.org/W2604165577","https://openalex.org/W2604314403","https://openalex.org/W2811271154","https://openalex.org/W2912805255","https://openalex.org/W2914866826","https://openalex.org/W2945623882","https://openalex.org/W2952436323","https://openalex.org/W2962756421","https://openalex.org/W2963265326","https://openalex.org/W2970641574","https://openalex.org/W2970986510","https://openalex.org/W3003265726","https://openalex.org/W3083368984","https://openalex.org/W3090656107","https://openalex.org/W3091231667","https://openalex.org/W3104415840","https://openalex.org/W3107653269","https://openalex.org/W3130909864","https://openalex.org/W3149839747","https://openalex.org/W3155001903","https://openalex.org/W3204971501","https://openalex.org/W3206893050","https://openalex.org/W4205589725","https://openalex.org/W4206478093","https://openalex.org/W4226297016","https://openalex.org/W4292779060","https://openalex.org/W4297183943","https://openalex.org/W4309673391","https://openalex.org/W4312751188","https://openalex.org/W4322706667","https://openalex.org/W4382396063","https://openalex.org/W4383737134","https://openalex.org/W4386310690","https://openalex.org/W4387267996","https://openalex.org/W4389084766","https://openalex.org/W6678830454","https://openalex.org/W6778883912","https://openalex.org/W6811710480","https://openalex.org/W6992478370"],"related_works":["https://openalex.org/W2088111589","https://openalex.org/W2321083758","https://openalex.org/W4366496304","https://openalex.org/W4229004718","https://openalex.org/W2375906709","https://openalex.org/W1988913758","https://openalex.org/W2350669386","https://openalex.org/W2348913748","https://openalex.org/W2381386433","https://openalex.org/W2045980693"],"abstract_inverted_index":{"Educational":[0],"content":[1,36,267],"recommendation":[2],"is":[3,146,185,225],"a":[4,96,104,113,197,205,234,242],"cornerstone":[5],"of":[6,53,64,82,89,161,221,230,237,245,258,288],"AI-enhanced":[7,289],"learning.":[8,290],"In":[9,72,118],"particular,":[10],"to":[11,33,120,174,178,212,271],"facilitate":[12],"navigating":[13],"the":[14,51,122,179,222,256,286],"diverse":[15],"learning":[16,20,28,274],"resources":[17],"available":[18,283],"on":[19,38,45,130,262,281],"platforms,":[21],"methods":[22,41],"are":[23,42,127],"needed":[24],"for":[25,55,69,133,154,227,249,265],"automatically":[26],"linking":[27],"materials,":[29],"e.g.,":[30],"in":[31,137,152,233,276],"order":[32],"recommend":[34],"textbook":[35],"based":[37,44,129,261],"exercises.":[39,162],"Such":[40],"typically":[43],"semantic":[46],"textual":[47],"similarity":[48],"(STS)":[49],"and":[50,108,158,182,239,241,247,251],"use":[52],"embeddings":[54,65,83,93,102,111,169,194,202,264],"text":[56,263],"representation.":[57],"However,":[58],"it":[59,210],"remains":[60],"unclear":[61],"what":[62],"types":[63,88,160,229],"should":[66],"be":[67],"used":[68],"this":[70,73],"task.":[71],"study,":[74],"we":[75],"carry":[76],"out":[77,148],"an":[78,138,171,277],"extensive":[79],"empirical":[80],"evaluation":[81,145],"derived":[84,203],"from":[85,103,112,170,204],"three":[86,155],"different":[87,131,156],"models:":[90],"(i)":[91],"static":[92,193],"trained":[94,195],"using":[95,149,167,196,201,259],"concept-based":[97],"knowledge":[98,198,216],"graph,":[99],"(ii)":[100],"contextual":[101,110,168],"pre-trained":[105,284],"language":[106,115,207],"model,":[107,208],"(iii)":[109],"large":[114],"model":[116,224],"(LLM).":[117],"addition":[119],"evaluating":[121],"models":[123,136],"individually,":[124],"various":[125],"ensembles":[126],"explored":[128],"strategies":[132],"combining":[134,190],"two":[135,159],"early":[139],"vs.":[140],"late":[141],"fusion":[142],"fashion.":[143],"The":[144,163,219,269],"carried":[147],"digital":[150,273],"textbooks":[151],"Swedish":[153],"subjects":[157],"results":[164],"show":[165],"that":[166,183],"LLM":[172],"leads":[173],"superior":[175],"performance":[176,220],"compared":[177],"other":[180],"models,":[181],"there":[184],"no":[186],"significant":[187],"improvement":[188],"when":[189],"these":[191],"with":[192,215],"graph.":[199],"When":[200],"smaller":[206],"however,":[209],"helps":[211],"combine":[213],"them":[214],"graph":[217],"embeddings.":[218],"best-performing":[223],"high":[226],"both":[228],"exercises,":[231],"resulting":[232],"mean":[235,243],"Recall@3":[236],"0.96":[238],"0.95":[240],"MRR":[244],"0.87":[246],"0.86":[248],"quizzes":[250],"study":[252],"questions,":[253],"respectively,":[254],"demonstrating":[255],"feasibility":[257],"STS":[260],"educational":[266],"recommendation.":[268],"ability":[270],"link":[272],"materials":[275],"unsupervised":[278],"manner\u2014relying":[279],"only":[280],"readily":[282],"models\u2014facilitates":[285],"development":[287]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":13}],"updated_date":"2026-06-04T09:04:59.091469","created_date":"2025-10-10T00:00:00"}