{"id":"https://openalex.org/W3194095485","doi":"https://doi.org/10.1145/3447548.3470810","title":"On the Power of Pre-Trained Text Representations","display_name":"On the Power of Pre-Trained Text Representations","publication_year":2021,"publication_date":"2021-08-12","ids":{"openalex":"https://openalex.org/W3194095485","doi":"https://doi.org/10.1145/3447548.3470810","mag":"3194095485"},"language":"en","primary_location":{"id":"doi:10.1145/3447548.3470810","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3470810","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100770786","display_name":"Meng Yu","orcid":"https://orcid.org/0000-0003-2554-2888"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yu Meng","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5046688345","display_name":"Jiaxin Huang","orcid":"https://orcid.org/0000-0001-8095-3343"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiaxin Huang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100433691","display_name":"Yu Zhang","orcid":"https://orcid.org/0000-0003-1100-4835"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu Zhang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019539533","display_name":"Jiawei Han","orcid":"https://orcid.org/0000-0002-3629-2696"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jiawei Han","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100770786"],"corresponding_institution_ids":["https://openalex.org/I157725225"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.11195824,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"4052","last_page":"4053"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8191460371017456},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.7307515144348145},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6817018985748291},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5916301608085632},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5668553709983826},{"id":"https://openalex.org/keywords/word-embedding","display_name":"Word embedding","score":0.5463708639144897},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5219968557357788},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.4944593906402588},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.46042418479919434},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.4491366147994995},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.42992785573005676},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3985433876514435},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.15369758009910583}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8191460371017456},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.7307515144348145},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6817018985748291},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5916301608085632},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5668553709983826},{"id":"https://openalex.org/C2777462759","wikidata":"https://www.wikidata.org/wiki/Q18395344","display_name":"Word embedding","level":3,"score":0.5463708639144897},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5219968557357788},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4944593906402588},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.46042418479919434},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4491366147994995},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.42992785573005676},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3985433876514435},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.15369758009910583},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3447548.3470810","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3447548.3470810","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 27th ACM SIGKDD Conference on Knowledge Discovery & Data Mining","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.8500000238418579}],"awards":[{"id":"https://openalex.org/G5829947576","display_name":null,"funder_award_id":"IIS-19-56151,IIS-17-41317,IIS-17-04532,2019897","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7391348854","display_name":null,"funder_award_id":"FA8750-19-2-1004,W911NF-17-C-0099","funder_id":"https://openalex.org/F4320332180","funder_display_name":"Defense Advanced Research Projects Agency"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1880262756","https://openalex.org/W2108281845","https://openalex.org/W2122678284","https://openalex.org/W2132827946","https://openalex.org/W2250539671","https://openalex.org/W2493916176","https://openalex.org/W2618798060","https://openalex.org/W2890931111","https://openalex.org/W2950133940","https://openalex.org/W2953014395","https://openalex.org/W2962739339","https://openalex.org/W2962936818","https://openalex.org/W2963341956","https://openalex.org/W2964311827","https://openalex.org/W2970597249","https://openalex.org/W2971324494","https://openalex.org/W3004119480","https://openalex.org/W3013571468","https://openalex.org/W3022907588","https://openalex.org/W3034715004","https://openalex.org/W3035055211","https://openalex.org/W3042602466","https://openalex.org/W3081051539","https://openalex.org/W3099045991","https://openalex.org/W3100474067","https://openalex.org/W3101606352","https://openalex.org/W3104217184","https://openalex.org/W3105538385","https://openalex.org/W3106109117","https://openalex.org/W3152519139","https://openalex.org/W3166913490","https://openalex.org/W3169884130","https://openalex.org/W3171416056","https://openalex.org/W4231510805"],"related_works":["https://openalex.org/W4288263119","https://openalex.org/W3015724364","https://openalex.org/W2967994095","https://openalex.org/W4285240985","https://openalex.org/W2900126711","https://openalex.org/W4286930972","https://openalex.org/W3202115945","https://openalex.org/W2542958340","https://openalex.org/W4389520438","https://openalex.org/W1991374750"],"abstract_inverted_index":{"Recent":[0],"years":[1],"have":[2,67],"witnessed":[3],"the":[4,42,69,107],"enormous":[5],"success":[6],"of":[7,15,45,111],"text":[8,16,65],"representation":[9],"learning":[10,22],"in":[11,96],"a":[12],"wide":[13],"range":[14],"mining":[17],"tasks.":[18],"Earlier":[19],"word":[20,35],"embedding":[21],"approaches":[23],"represent":[24],"words":[25],"as":[26,41,99],"fixed":[27],"low-dimensional":[28],"vectors":[29],"to":[30,86,103],"capture":[31],"their":[32],"semantics.":[33],"The":[34],"embeddings":[36],"so":[37],"learned":[38],"are":[39],"used":[40],"input":[43],"features":[44,81],"task-specific":[46,94],"models.":[47],"Recently,":[48],"pre-trained":[49,76],"language":[50,56,71],"models":[51,62,95],"(PLMs),":[52],"which":[53],"learn":[54],"universal":[55],"representations":[57,77],"via":[58],"pre-training":[59],"Transformer-based":[60],"neural":[61],"on":[63,106],"large-scale":[64],"corpora,":[66],"revolutionized":[68],"natural":[70],"processing":[72],"(NLP)":[73],"field.":[74],"Such":[75],"encode":[78],"generic":[79],"linguistic":[80],"that":[82],"can":[83],"be":[84,104],"transferred":[85],"almost":[87],"any":[88],"text-related":[89],"applications.":[90],"PLMs":[91],"outperform":[92],"previous":[93],"many":[97],"applications":[98],"they":[100],"only":[101],"need":[102],"fine-tuned":[105],"target":[108],"corpus":[109],"instead":[110],"being":[112],"trained":[113],"from":[114],"scratch.":[115]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}