{"id":"https://openalex.org/W3190572136","doi":"https://doi.org/10.48550/arxiv.2108.05542","title":"AMMUS : A Survey of Transformer-based Pretrained Models in Natural Language Processing","display_name":"AMMUS : A Survey of Transformer-based Pretrained Models in Natural Language Processing","publication_year":2021,"publication_date":"2021-08-12","ids":{"openalex":"https://openalex.org/W3190572136","doi":"https://doi.org/10.48550/arxiv.2108.05542","mag":"3190572136"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2108.05542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.05542","pdf_url":"https://arxiv.org/pdf/2108.05542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2108.05542","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007480962","display_name":"Katikapalli Subramanyam Kalyan","orcid":"https://orcid.org/0000-0002-5490-6444","observed_orcids":["https://orcid.org/0000-0002-5490-6444"]},"institutions":[{"id":"https://openalex.org/I122964287","display_name":"National Institute of Technology Tiruchirappalli","ror":"https://ror.org/047x65e68","country_code":"IN","type":"education","lineage":["https://openalex.org/I122964287"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Kalyan, Katikapalli Subramanyam","raw_affiliation_strings":["Department of Computer Applications , National Institute of Technology Trichy , Trichy , Tamil Nadu , India , 620015"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Applications , National Institute of Technology Trichy , Trichy , Tamil Nadu , India , 620015","institution_ids":["https://openalex.org/I122964287"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5006853359","display_name":"Ajit Rajasekharan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I122964287","display_name":"National Institute of Technology Tiruchirappalli","ror":"https://ror.org/047x65e68","country_code":"IN","type":"education","lineage":["https://openalex.org/I122964287"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Rajasekharan, Ajit","raw_affiliation_strings":["Cambridge , MA , USA, 02142","Department of Computer Applications , National Institute of Technology Trichy , Trichy , Tamil Nadu , India , 620015"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Cambridge , MA , USA, 02142","institution_ids":[]},{"raw_affiliation_string":"Department of Computer Applications , National Institute of Technology Trichy , Trichy , Tamil Nadu , India , 620015","institution_ids":["https://openalex.org/I122964287"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007535678","display_name":"S. Sangeetha","orcid":"https://orcid.org/0000-0001-6630-1664","observed_orcids":["https://orcid.org/0000-0001-6630-1664"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sangeetha, Sivanesan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":360,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.003800000064074993,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.9559999704360962},{"id":"https://openalex.org/keywords/transfer-learning","display_name":"transfer learning","score":0.9129999876022339},{"id":"https://openalex.org/keywords/embeddings","display_name":"embeddings","score":0.6370000243186951},{"id":"https://openalex.org/keywords/benchmarking","display_name":"benchmarking","score":0.47600001096725464},{"id":"https://openalex.org/keywords/gpt","display_name":"GPT","score":0.8130000233650208},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":1.0}],"concepts":[{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.8191426396369934},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7593618631362915},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.5744280815124512},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5363729000091553},{"id":"https://openalex.org/C2776207758","wikidata":"https://www.wikidata.org/wiki/Q5303302","display_name":"Downstream (manufacturing)","level":2,"score":0.4896686375141144},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.44702956080436707},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.44696637988090515},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.41559380292892456},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3991280496120453},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07920962572097778},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:arXiv.org:2108.05542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.05542","pdf_url":"https://arxiv.org/pdf/2108.05542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3190572136","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2108.05542.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2108.05542","doi":"https://doi.org/10.48550/arxiv.2108.05542","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2108.05542","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2108.05542","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2108.05542","pdf_url":"https://arxiv.org/pdf/2108.05542","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3190572136.pdf","grobid_xml":"https://content.openalex.org/works/W3190572136.grobid-xml"},"referenced_works_count":307,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1614298861","https://openalex.org/W1670132599","https://openalex.org/W1686810756","https://openalex.org/W1821462560","https://openalex.org/W2107901333","https://openalex.org/W2117539524","https://openalex.org/W2120615054","https://openalex.org/W2134797427","https://openalex.org/W2157331557","https://openalex.org/W2159583324","https://openalex.org/W2163605009","https://openalex.org/W2165698076","https://openalex.org/W2183341477","https://openalex.org/W2194775991","https://openalex.org/W2250539671","https://openalex.org/W2250770256","https://openalex.org/W2294370754","https://openalex.org/W2396881363","https://openalex.org/W2407776548","https://openalex.org/W2525778437","https://openalex.org/W2572474373","https://openalex.org/W2606321545","https://openalex.org/W2626778328","https://openalex.org/W2785611959","https://openalex.org/W2787560479","https://openalex.org/W2806758205","https://openalex.org/W2866343820","https://openalex.org/W2880875857","https://openalex.org/W2885185669","https://openalex.org/W2897105542","https://openalex.org/W2898700502","https://openalex.org/W2911300548","https://openalex.org/W2911489562","https://openalex.org/W2933138175","https://openalex.org/W2944815030","https://openalex.org/W2945260553","https://openalex.org/W2945767825","https://openalex.org/W2945785363","https://openalex.org/W2946676565","https://openalex.org/W2946794439","https://openalex.org/W2946948417","https://openalex.org/W2947415936","https://openalex.org/W2948096105","https://openalex.org/W2948223045","https://openalex.org/W2948902769","https://openalex.org/W2950813464","https://openalex.org/W2953356739","https://openalex.org/W2960374072","https://openalex.org/W2962753370","https://openalex.org/W2962784628","https://openalex.org/W2963121782","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2963716420","https://openalex.org/W2963748441","https://openalex.org/W2963854351","https://openalex.org/W2963855739","https://openalex.org/W2963877297","https://openalex.org/W2965373594","https://openalex.org/W2966892770","https://openalex.org/W2966989210","https://openalex.org/W2968289784","https://openalex.org/W2969515962","https://openalex.org/W2969624041","https://openalex.org/W2970049541","https://openalex.org/W2970290563","https://openalex.org/W2970476646","https://openalex.org/W2970636124","https://openalex.org/W2970641574","https://openalex.org/W2970771982","https://openalex.org/W2970986510","https://openalex.org/W2971008823","https://openalex.org/W2971196067","https://openalex.org/W2971207485","https://openalex.org/W2971258845","https://openalex.org/W2973071945","https://openalex.org/W2975059944","https://openalex.org/W2975185270","https://openalex.org/W2975381464","https://openalex.org/W2978017171","https://openalex.org/W2979314664","https://openalex.org/W2979694518","https://openalex.org/W2981852735","https://openalex.org/W2982399380","https://openalex.org/W2983040767","https://openalex.org/W2983772459","https://openalex.org/W2985509622","https://openalex.org/W2986154550","https://openalex.org/W2990188683","https://openalex.org/W2990704537","https://openalex.org/W2991382858","https://openalex.org/W2994673210","https://openalex.org/W2994980856","https://openalex.org/W2995647371","https://openalex.org/W2996580882","https://openalex.org/W2996822578","https://openalex.org/W2997710335","https://openalex.org/W2998183051","https://openalex.org/W2998554035","https://openalex.org/W2998653236","https://openalex.org/W2999168658","https://openalex.org/W3000965575","https://openalex.org/W3001279689","https://openalex.org/W3001434439","https://openalex.org/W3002832564","https://openalex.org/W3005700362","https://openalex.org/W3007007518","https://openalex.org/W3007955273","https://openalex.org/W3008110149","https://openalex.org/W3008219293","https://openalex.org/W3010108619","https://openalex.org/W3011159643","https://openalex.org/W3011279327","https://openalex.org/W3011574394","https://openalex.org/W3011718307","https://openalex.org/W3012393889","https://openalex.org/W3013838212","https://openalex.org/W3013840636","https://openalex.org/W3015233032","https://openalex.org/W3015453090","https://openalex.org/W3015468748","https://openalex.org/W3016187590","https://openalex.org/W3017022649","https://openalex.org/W3017231514","https://openalex.org/W3017549762","https://openalex.org/W3017637887","https://openalex.org/W3017961061","https://openalex.org/W3021191241","https://openalex.org/W3021457404","https://openalex.org/W3022112973","https://openalex.org/W3022717752","https://openalex.org/W3022810465","https://openalex.org/W3023437581","https://openalex.org/W3023805054","https://openalex.org/W3024308166","https://openalex.org/W3025064182","https://openalex.org/W3026732421","https://openalex.org/W3027440908","https://openalex.org/W3028836324","https://openalex.org/W3029760648","https://openalex.org/W3030163527","https://openalex.org/W3032299950","https://openalex.org/W3032816972","https://openalex.org/W3033187248","https://openalex.org/W3033862527","https://openalex.org/W3034255912","https://openalex.org/W3034457371","https://openalex.org/W3034715004","https://openalex.org/W3035579820","https://openalex.org/W3035725276","https://openalex.org/W3037063616","https://openalex.org/W3037252472","https://openalex.org/W3038035611","https://openalex.org/W3040245432","https://openalex.org/W3040573126","https://openalex.org/W3046375318","https://openalex.org/W3047171714","https://openalex.org/W3048823912","https://openalex.org/W3081168214","https://openalex.org/W3081210419","https://openalex.org/W3081505754","https://openalex.org/W3085177480","https://openalex.org/W3086007799","https://openalex.org/W3089168780","https://openalex.org/W3091355780","https://openalex.org/W3092129462","https://openalex.org/W3092292868","https://openalex.org/W3093553144","https://openalex.org/W3093699284","https://openalex.org/W3093814160","https://openalex.org/W3094002940","https://openalex.org/W3094446431","https://openalex.org/W3095224541","https://openalex.org/W3095609828","https://openalex.org/W3095642204","https://openalex.org/W3095771422","https://openalex.org/W3096266342","https://openalex.org/W3096403953","https://openalex.org/W3096565276","https://openalex.org/W3098300729","https://openalex.org/W3098466758","https://openalex.org/W3098469895","https://openalex.org/W3098576111","https://openalex.org/W3098605233","https://openalex.org/W3098649723","https://openalex.org/W3099008231","https://openalex.org/W3099782249","https://openalex.org/W3099919888","https://openalex.org/W3100652389","https://openalex.org/W3101045333","https://openalex.org/W3102046836","https://openalex.org/W3102483398","https://openalex.org/W3104163040","https://openalex.org/W3104578551","https://openalex.org/W3104613728","https://openalex.org/W3105111366","https://openalex.org/W3105238007","https://openalex.org/W3105601320","https://openalex.org/W3105867580","https://openalex.org/W3105966348","https://openalex.org/W3106339673","https://openalex.org/W3109919947","https://openalex.org/W3110048753","https://openalex.org/W3110457105","https://openalex.org/W3112689365","https://openalex.org/W3114651185","https://openalex.org/W3115462295","https://openalex.org/W3116295307","https://openalex.org/W3116407756","https://openalex.org/W3116423158","https://openalex.org/W3116527904","https://openalex.org/W3118043957","https://openalex.org/W3118580427","https://openalex.org/W3119546299","https://openalex.org/W3119866685","https://openalex.org/W3119997354","https://openalex.org/W3120094169","https://openalex.org/W3120253119","https://openalex.org/W3120490999","https://openalex.org/W3122838366","https://openalex.org/W3125056032","https://openalex.org/W3125826128","https://openalex.org/W3125833936","https://openalex.org/W3126675481","https://openalex.org/W3127389184","https://openalex.org/W3127729136","https://openalex.org/W3128029819","https://openalex.org/W3130716829","https://openalex.org/W3135427360","https://openalex.org/W3136221257","https://openalex.org/W3136363192","https://openalex.org/W3139080614","https://openalex.org/W3141023492","https://openalex.org/W3152609875","https://openalex.org/W3152879179","https://openalex.org/W3152956381","https://openalex.org/W3153124855","https://openalex.org/W3153266325","https://openalex.org/W3155121067","https://openalex.org/W3155904893","https://openalex.org/W3156371424","https://openalex.org/W3156404059","https://openalex.org/W3156636935","https://openalex.org/W3156665996","https://openalex.org/W3158196282","https://openalex.org/W3158631574","https://openalex.org/W3158835769","https://openalex.org/W3159134453","https://openalex.org/W3159795318","https://openalex.org/W3160137267","https://openalex.org/W3162296828","https://openalex.org/W3164045210","https://openalex.org/W3164211145","https://openalex.org/W3164540570","https://openalex.org/W3164896303","https://openalex.org/W3168772426","https://openalex.org/W3169113923","https://openalex.org/W3169483174","https://openalex.org/W3170092793","https://openalex.org/W3170647102","https://openalex.org/W3171649327","https://openalex.org/W3171975879","https://openalex.org/W3172021172","https://openalex.org/W3172198372","https://openalex.org/W3172205429","https://openalex.org/W3173777717","https://openalex.org/W3173814445","https://openalex.org/W3173870405","https://openalex.org/W3174234060","https://openalex.org/W3174418826","https://openalex.org/W3174770825","https://openalex.org/W3175362188","https://openalex.org/W3175870271","https://openalex.org/W3175898847","https://openalex.org/W3176617251","https://openalex.org/W3176685217","https://openalex.org/W3177765786","https://openalex.org/W3178814223","https://openalex.org/W3182414670","https://openalex.org/W3183962691","https://openalex.org/W3185341429","https://openalex.org/W3186655327","https://openalex.org/W3186870698","https://openalex.org/W3187418919","https://openalex.org/W3197040386","https://openalex.org/W3206841862","https://openalex.org/W3212386673","https://openalex.org/W3212554815","https://openalex.org/W3213458975","https://openalex.org/W3213730158","https://openalex.org/W4224281327","https://openalex.org/W4246183800","https://openalex.org/W4287077924","https://openalex.org/W4287200699","https://openalex.org/W4287484552","https://openalex.org/W4287545908","https://openalex.org/W4287548279","https://openalex.org/W4287778518","https://openalex.org/W4290875442","https://openalex.org/W4300427991","https://openalex.org/W4303939357","https://openalex.org/W4304690179","https://openalex.org/W4324148013","https://openalex.org/W4324392584","https://openalex.org/W4385573665","https://openalex.org/W6893675597"],"related_works":["https://openalex.org/W4223543859","https://openalex.org/W3011574394","https://openalex.org/W4327737695","https://openalex.org/W4403126832","https://openalex.org/W4291636998","https://openalex.org/W4409045841","https://openalex.org/W4407768809","https://openalex.org/W3157252883","https://openalex.org/W3198659451","https://openalex.org/W4382246105"],"abstract_inverted_index":{"Transformer-based":[0],"pretrained":[1],"language":[2,42],"models":[3,18,25,61,74],"(T-PTLMs)":[4],"have":[5],"achieved":[6],"great":[7],"success":[8],"in":[9,185],"almost":[10],"every":[11],"NLP":[12],"task.":[13],"The":[14],"evolution":[15],"of":[16,31,47,72,88,114,121,133,145],"these":[17,154],"started":[19],"with":[20,139,181],"GPT":[21],"and":[22,35,53,104,116,127],"BERT.":[23],"These":[24,60],"are":[26],"built":[27],"on":[28],"the":[29,146,172,182],"top":[30],"transformers,":[32],"self-supervised":[33,51,89],"learning":[34,52],"transfer":[36,54],"learning.":[37,90],"Transformed-based":[38],"PTLMs":[39],"learn":[40,171],"universal":[41],"representations":[43],"from":[44,75],"large":[45],"volumes":[46],"text":[48],"data":[49],"using":[50],"this":[55,78,160],"knowledge":[56,65],"to":[57,66,137,170,178],"downstream":[58,67,73,105],"tasks.":[59],"provide":[62],"good":[63,168],"background":[64],"tasks":[68],"which":[69,150],"avoids":[70],"training":[71],"scratch.":[76],"In":[77],"comprehensive":[79,161],"survey":[80,162],"paper,":[81],"we":[82,92,109,142],"initially":[83],"give":[84,118],"a":[85,111,131,167],"brief":[86,119],"overview":[87,120],"Next,":[91,108],"explain":[93],"various":[94,122,134],"core":[95,173],"concepts":[96,174],"like":[97],"pretraining,":[98],"pretraining":[99,101],"methods,":[100],"tasks,":[102],"embeddings":[103],"adaptation":[106],"methods.":[107],"present":[110,130],"new":[112],"taxonomy":[113],"T-PTLMs":[115],"then":[117],"benchmarks":[123],"including":[124],"both":[125],"intrinsic":[126],"extrinsic.":[128],"We":[129,156],"summary":[132],"useful":[135],"libraries":[136],"work":[138],"T-PTLMs.":[140,186],"Finally,":[141],"highlight":[143],"some":[144],"future":[147],"research":[148],"directions":[149],"will":[151,164],"further":[152],"improve":[153],"models.":[155],"strongly":[157],"believe":[158],"that":[159],"paper":[163],"serve":[165],"as":[166,175,177],"reference":[169],"well":[176],"stay":[179],"updated":[180],"recent":[183],"happenings":[184]},"counts_by_year":[{"year":2026,"cited_by_count":31},{"year":2025,"cited_by_count":71},{"year":2024,"cited_by_count":114},{"year":2023,"cited_by_count":96},{"year":2022,"cited_by_count":40},{"year":2021,"cited_by_count":8}],"updated_date":"2026-11-11T12:16:26.678958","created_date":"2021-08-16T00:00:00"}