{"id":"https://openalex.org/W4386275705","doi":"https://doi.org/10.1109/tkde.2023.3310002","title":"A Survey of Knowledge Enhanced Pre-Trained Language Models","display_name":"A Survey of Knowledge Enhanced Pre-Trained Language Models","publication_year":2023,"publication_date":"2023-08-30","ids":{"openalex":"https://openalex.org/W4386275705","doi":"https://doi.org/10.1109/tkde.2023.3310002"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2023.3310002","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2023.3310002","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5063307425","display_name":"Linmei Hu","orcid":"https://orcid.org/0000-0003-0993-8495"},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Linmei Hu","raw_affiliation_strings":["School of Computer Science and Technology, Beijing Institute of Technology, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Beijing Institute of Technology, Beijing, China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081789111","display_name":"Zeyi Liu","orcid":"https://orcid.org/0000-0002-5691-6849"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyi Liu","raw_affiliation_strings":["School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5042424428","display_name":"Ziwang Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ziwang Zhao","raw_affiliation_strings":["School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060498828","display_name":"Lei Hou","orcid":"https://orcid.org/0000-0002-8907-3526"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lei Hou","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038612499","display_name":"Liqiang Nie","orcid":"https://orcid.org/0000-0003-1476-0273"},"institutions":[{"id":"https://openalex.org/I158809036","display_name":"Shenzhen Institute of Information Technology","ror":"https://ror.org/03wrf9427","country_code":"CN","type":"education","lineage":["https://openalex.org/I158809036"]},{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liqiang Nie","raw_affiliation_strings":["School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), Shenzhen, Guangdong, China"],"affiliations":[{"raw_affiliation_string":"School of Computer Science and Technology, Harbin Institute of Technology (Shenzhen), Shenzhen, Guangdong, China","institution_ids":["https://openalex.org/I158809036","https://openalex.org/I204983213"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5003324011","display_name":"Juanzi Li","orcid":"https://orcid.org/0000-0002-6244-0664"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Juanzi Li","raw_affiliation_strings":["Department of Computer Science and Technology, Tsinghua University, Beijing, China"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Technology, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5063307425"],"corresponding_institution_ids":["https://openalex.org/I125839683"],"apc_list":null,"apc_paid":null,"fwci":25.0548,"has_fulltext":false,"cited_by_count":144,"citation_normalized_percentile":{"value":0.99717213,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"36","issue":"4","first_page":"1413","last_page":"1430"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8227677345275879},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6255888938903809},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.623629629611969},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5663223266601562},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.5451099872589111},{"id":"https://openalex.org/keywords/knowledge-graph","display_name":"Knowledge graph","score":0.47530943155288696},{"id":"https://openalex.org/keywords/domain-knowledge","display_name":"Domain knowledge","score":0.4664328396320343},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.45548975467681885},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.4406622350215912},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3955153524875641}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8227677345275879},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6255888938903809},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.623629629611969},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5663223266601562},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5451099872589111},{"id":"https://openalex.org/C2987255567","wikidata":"https://www.wikidata.org/wiki/Q33002955","display_name":"Knowledge graph","level":2,"score":0.47530943155288696},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.4664328396320343},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.45548975467681885},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.4406622350215912},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3955153524875641}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2023.3310002","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2023.3310002","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7599999904632568}],"awards":[{"id":"https://openalex.org/G2815490039","display_name":null,"funder_award_id":"62276029","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":148,"referenced_works":["https://openalex.org/W1713614699","https://openalex.org/W2144578941","https://openalex.org/W2466175319","https://openalex.org/W2743151379","https://openalex.org/W2759136286","https://openalex.org/W2759211898","https://openalex.org/W2888482885","https://openalex.org/W2890894339","https://openalex.org/W2891826200","https://openalex.org/W2896457183","https://openalex.org/W2906152891","https://openalex.org/W2911489562","https://openalex.org/W2912924812","https://openalex.org/W2923014074","https://openalex.org/W2938830017","https://openalex.org/W2939137134","https://openalex.org/W2947337775","https://openalex.org/W2953356739","https://openalex.org/W2962739339","https://openalex.org/W2962891712","https://openalex.org/W2963026768","https://openalex.org/W2963339397","https://openalex.org/W2963777632","https://openalex.org/W2963929190","https://openalex.org/W2964303116","https://openalex.org/W2970476646","https://openalex.org/W2970771982","https://openalex.org/W2970986510","https://openalex.org/W2998385486","https://openalex.org/W3006087551","https://openalex.org/W3010336026","https://openalex.org/W3014521650","https://openalex.org/W3015453090","https://openalex.org/W3034555021","https://openalex.org/W3034999214","https://openalex.org/W3035153870","https://openalex.org/W3035419191","https://openalex.org/W3035725276","https://openalex.org/W3088754460","https://openalex.org/W3090656107","https://openalex.org/W3090716330","https://openalex.org/W3092288641","https://openalex.org/W3098267758","https://openalex.org/W3099178230","https://openalex.org/W3102187933","https://openalex.org/W3102844651","https://openalex.org/W3104007871","https://openalex.org/W3104415840","https://openalex.org/W3105601320","https://openalex.org/W3111372685","https://openalex.org/W3113425182","https://openalex.org/W3114916066","https://openalex.org/W3115729981","https://openalex.org/W3129831491","https://openalex.org/W3151929433","https://openalex.org/W3153427360","https://openalex.org/W3153543512","https://openalex.org/W3154449322","https://openalex.org/W3156789018","https://openalex.org/W3169283738","https://openalex.org/W3169726359","https://openalex.org/W3173169192","https://openalex.org/W3173220653","https://openalex.org/W3173909648","https://openalex.org/W3174413662","https://openalex.org/W3174770825","https://openalex.org/W3175034459","https://openalex.org/W3175604467","https://openalex.org/W3176175717","https://openalex.org/W3176750236","https://openalex.org/W3176929804","https://openalex.org/W3182414949","https://openalex.org/W3185341429","https://openalex.org/W3186545525","https://openalex.org/W3188542058","https://openalex.org/W3194836374","https://openalex.org/W3198571508","https://openalex.org/W3198659451","https://openalex.org/W3201662444","https://openalex.org/W3207796810","https://openalex.org/W3207976211","https://openalex.org/W3210129272","https://openalex.org/W3214536449","https://openalex.org/W3217374296","https://openalex.org/W4200629408","https://openalex.org/W4205291549","https://openalex.org/W4205513945","https://openalex.org/W4211060471","https://openalex.org/W4221141423","https://openalex.org/W4221157571","https://openalex.org/W4224308101","https://openalex.org/W4224920984","https://openalex.org/W4226063512","https://openalex.org/W4226085996","https://openalex.org/W4226297016","https://openalex.org/W4226455589","https://openalex.org/W4242082217","https://openalex.org/W4283026156","https://openalex.org/W4285109111","https://openalex.org/W4285122897","https://openalex.org/W4285151662","https://openalex.org/W4285169833","https://openalex.org/W4285233612","https://openalex.org/W4285601776","https://openalex.org/W4285606726","https://openalex.org/W4287855173","https://openalex.org/W4287890147","https://openalex.org/W4288089799","https://openalex.org/W4296878971","https://openalex.org/W4309811444","https://openalex.org/W4312805275","https://openalex.org/W4322718421","https://openalex.org/W4362515116","https://openalex.org/W4385245566","https://openalex.org/W4385572953","https://openalex.org/W4385573097","https://openalex.org/W4385573136","https://openalex.org/W4385573708","https://openalex.org/W4385573862","https://openalex.org/W6691459498","https://openalex.org/W6691476020","https://openalex.org/W6731780175","https://openalex.org/W6755699846","https://openalex.org/W6755829550","https://openalex.org/W6761910064","https://openalex.org/W6762122294","https://openalex.org/W6762392948","https://openalex.org/W6769627184","https://openalex.org/W6770813465","https://openalex.org/W6770982027","https://openalex.org/W6771543990","https://openalex.org/W6777615688","https://openalex.org/W6778883912","https://openalex.org/W6779857854","https://openalex.org/W6783708288","https://openalex.org/W6783976220","https://openalex.org/W6798398338","https://openalex.org/W6800480908","https://openalex.org/W6802645015","https://openalex.org/W6803181477","https://openalex.org/W6810081322","https://openalex.org/W6810152935","https://openalex.org/W6810730852","https://openalex.org/W6838461927","https://openalex.org/W6838811116","https://openalex.org/W6846539114","https://openalex.org/W6850359928","https://openalex.org/W6851775633"],"related_works":["https://openalex.org/W3034800807","https://openalex.org/W3035130722","https://openalex.org/W4287758468","https://openalex.org/W3034878914","https://openalex.org/W4310731959","https://openalex.org/W4292070284","https://openalex.org/W3102948683","https://openalex.org/W3019688513","https://openalex.org/W4312933959","https://openalex.org/W4391009500"],"abstract_inverted_index":{"Pre-trained":[0,92],"Language":[1,24,93,112,117],"Models":[2,94],"(PLMs)":[3],"which":[4],"are":[5,153],"trained":[6],"on":[7,19],"large":[8],"text":[9,42,141],"corpus":[10],"via":[11],"self-supervised":[12],"learning":[13],"method,":[14],"have":[15,53],"yielded":[16],"promising":[17,165],"performance":[18],"various":[20],"tasks":[21,46,125],"in":[22],"Natural":[23,111,116],"Processing":[25],"(NLP).":[26],"However,":[27],"though":[28],"PLMs":[29,76],"with":[30],"huge":[31],"parameters":[32],"can":[33],"effectively":[34],"possess":[35],"rich":[36],"knowledge":[37,74,135,143],"learned":[38],"from":[39],"massive":[40],"training":[41],"and":[43,115,146,157],"benefit":[44],"downstream":[45],"at":[47],"the":[48,63,132],"fine-tuning":[49],"stage,":[50],"they":[51],"still":[52],"some":[54,164],"limitations":[55],"such":[56],"as":[57],"poor":[58],"reasoning":[59],"ability":[60],"due":[61],"to":[62,72,77,96,120],"lack":[64],"of":[65,89,126,134,168],"external":[66],"knowledge.":[67,148],"Research":[68],"has":[69],"been":[70],"dedicated":[71],"incorporating":[73],"into":[75,101,136,155],"tackle":[78],"these":[79,122],"issues.":[80],"In":[81],"this":[82,102],"paper,":[83],"we":[84,130,161],"present":[85],"a":[86,98],"comprehensive":[87],"review":[88],"Knowledge":[90],"Enhanced":[91],"(KE-PLMs)":[95],"provide":[97],"clear":[99],"insight":[100],"thriving":[103],"field.":[104],"We":[105],"introduce":[106],"appropriate":[107],"taxonomies":[108],"respectively":[109],"for":[110,151],"Understanding":[113],"(NLU)":[114],"Generation":[118],"(NLG)":[119],"highlight":[121],"two":[123],"main":[124],"NLP.":[127],"For":[128],"NLU,":[129],"divide":[131],"types":[133],"four":[137],"categories:":[138],"linguistic":[139],"knowledge,":[140,142],"graph":[144],"(KG),":[145],"rule":[147],"The":[149],"KE-PLMs":[150],"NLG":[152],"categorized":[154],"KG-based":[156],"retrieval-based":[158],"methods.":[159],"Finally,":[160],"point":[162],"out":[163],"future":[166],"directions":[167],"KE-PLMs.":[169]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":73},{"year":2024,"cited_by_count":46},{"year":2023,"cited_by_count":11}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}