{"id":"https://openalex.org/W3137214022","doi":null,"title":"All NLP Tasks Are Generation Tasks: A General Pretraining Framework","display_name":"All NLP Tasks Are Generation Tasks: A General Pretraining Framework","publication_year":2021,"publication_date":"2021-03-18","ids":{"openalex":"https://openalex.org/W3137214022","mag":"3137214022"},"language":"en","primary_location":{"id":"mag:3137214022","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2103.10360.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2103.10360.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5018336656","display_name":"Zhengxiao Du","orcid":"https://orcid.org/0000-0002-8223-4147","observed_orcids":["https://orcid.org/0000-0002-8223-4147"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhengxiao Du","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-8223-4147","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101546902","display_name":"Yujie Qian","orcid":"https://orcid.org/0000-0003-3747-4552","observed_orcids":["https://orcid.org/0000-0003-3747-4552"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yujie Qian","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-3747-4552","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100441094","display_name":"Xiao Liu","orcid":"https://orcid.org/0000-0002-9226-4569","observed_orcids":["https://orcid.org/0000-0002-9226-4569","https://orcid.org/0009-0006-3387-2317"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xiao Liu","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-9226-4569","affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Ming Ding","orcid":"https://orcid.org/0000-0002-6376-3174","observed_orcids":["https://orcid.org/0000-0002-6376-3174"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ming Ding","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-6376-3174","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083333630","display_name":"Jiezhong Qiu","orcid":"https://orcid.org/0000-0001-9514-0708","observed_orcids":["https://orcid.org/0000-0001-9514-0708"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiezhong Qiu","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-9514-0708","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050303775","display_name":"Zhilin Yang","orcid":"https://orcid.org/0000-0003-0944-4903","observed_orcids":["https://orcid.org/0000-0003-0944-4903","https://orcid.org/0000-0002-8598-922X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhilin Yang","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-0944-4903","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5044791875","display_name":"Jie Tang","orcid":"https://orcid.org/0000-0003-3487-4593","observed_orcids":["https://orcid.org/0000-0003-3487-4593"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jie Tang","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0003-3487-4593","affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":38,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9932000041007996,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.004000000189989805,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0010000000474974513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/glm","display_name":"GLM","score":0.5529999732971191},{"id":"https://openalex.org/keywords/conditional-generation","display_name":"conditional generation","score":0.49300000071525574},{"id":"https://openalex.org/keywords/unconditional-generation","display_name":"unconditional generation","score":0.9990000128746033},{"id":"https://openalex.org/keywords/text-classification","display_name":"text classification","score":0.7870000004768372},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"natural language understanding","score":0.8019999861717224},{"id":"https://openalex.org/keywords/superglue","display_name":"SuperGLUE","score":0.7120000123977661},{"id":"https://openalex.org/keywords/bert-comparison","display_name":"BERT comparison","score":0.6830000281333923},{"id":"https://openalex.org/keywords/autoregressive-models","display_name":"autoregressive models","score":0.5619999766349792},{"id":"https://openalex.org/keywords/encoder-decoder-architecture","display_name":"encoder-decoder architecture","score":0.9049999713897705}],"concepts":[{"id":"https://openalex.org/C27158222","wikidata":"https://www.wikidata.org/wiki/Q5532422","display_name":"Generalizability theory","level":2,"score":0.8483273983001709},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8036768436431885},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7172771096229553},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6502686142921448},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.5572656393051147},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.542312741279602},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5344457030296326},{"id":"https://openalex.org/C159877910","wikidata":"https://www.wikidata.org/wiki/Q2202883","display_name":"Autoregressive model","level":2,"score":0.5062798261642456},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.48793885111808777},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4709167182445526},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4382251799106598},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.4311860501766205},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.412240594625473},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.29354527592658997},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.11819401383399963},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0766785740852356},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3137214022","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2103.10360.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3137214022","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2103.10360.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W1599016936","https://openalex.org/W1843891098","https://openalex.org/W2145755360","https://openalex.org/W2153579005","https://openalex.org/W2250539671","https://openalex.org/W2473344385","https://openalex.org/W2525127255","https://openalex.org/W2581377246","https://openalex.org/W2626778328","https://openalex.org/W2804897457","https://openalex.org/W2805206884","https://openalex.org/W2898662126","https://openalex.org/W2908510526","https://openalex.org/W2944815030","https://openalex.org/W2945260553","https://openalex.org/W2946659172","https://openalex.org/W2950813464","https://openalex.org/W2962753370","https://openalex.org/W2963341956","https://openalex.org/W2965373594","https://openalex.org/W2990704537","https://openalex.org/W3002104146","https://openalex.org/W3004437686","https://openalex.org/W3007759824","https://openalex.org/W3024131638","https://openalex.org/W3082274269","https://openalex.org/W3085177480","https://openalex.org/W3094045953","https://openalex.org/W3098903812","https://openalex.org/W3106339673","https://openalex.org/W3106340866","https://openalex.org/W3120337237"],"related_works":["https://openalex.org/W4287265459","https://openalex.org/W3110048753","https://openalex.org/W2945260553","https://openalex.org/W4221149883","https://openalex.org/W4312091226","https://openalex.org/W4283650355","https://openalex.org/W4382246105","https://openalex.org/W2950813464","https://openalex.org/W4221158424","https://openalex.org/W3214020110"],"abstract_inverted_index":{"There":[0],"have":[1],"been":[2],"various":[3],"types":[4],"of":[5,45,146,154],"pretraining":[6,47,67],"architectures":[7],"including":[8],"autoregressive":[9],"models":[10,14,19,108],"(e.g.,":[11,15,20],"GPT),":[12],"autoencoding":[13],"BERT),":[16],"and":[17,40,61,95,163],"encoder-decoder":[18],"T5).":[21],"On":[22],"the":[23,46,50,136,143,157,167],"other":[24],"hand,":[25],"NLP":[26],"tasks":[27,98],"are":[28],"different":[29,175],"in":[30,160],"nature,":[31],"with":[32,99,142,151],"three":[33,84],"main":[34],"categories":[35],"being":[36],"classification,":[37,92],"unconditional":[38,93,164],"generation,":[39,94],"conditional":[41,96,162],"generation.":[42],"However,":[43],"none":[44],"frameworks":[48],"performs":[49,89],"best":[51,158],"for":[52,58,126],"all":[53],"tasks,":[54],"which":[55,123,170],"introduces":[56],"inconvenience":[57],"model":[59],"development":[60],"selection.":[62],"We":[63],"propose":[64],"a":[65],"novel":[66],"framework":[68],"GLM":[69,131,150],"(General":[70],"Language":[71],"Model)":[72],"to":[73,78,112,174],"address":[74],"this":[75],"challenge.":[76],"Compared":[77],"previous":[79],"work,":[80],"our":[81],"architecture":[82],"has":[83],"major":[85],"benefits:":[86],"(1)":[87],"it":[88,105,117],"well":[90],"on":[91,109,135],"generation":[97,165],"one":[100],"single":[101],"pretrained":[102],"model;":[103],"(2)":[104],"outperforms":[106,133],"BERT-like":[107],"classification":[110],"due":[111],"improved":[113],"pretrain-finetune":[114],"consistency;":[115],"(3)":[116],"naturally":[118],"handles":[119],"variable-length":[120],"blank":[121],"filling":[122],"is":[124],"crucial":[125],"many":[127],"downstream":[128,176],"tasks.":[129,177],"Empirically,":[130],"substantially":[132],"BERT":[134],"SuperGLUE":[137],"natural":[138],"language":[139],"understanding":[140],"benchmark":[141],"same":[144,168],"amount":[145],"pre-training":[147],"data.":[148],"Moreover,":[149],"1.25x":[152],"parameters":[153],"BERT-Large":[155],"achieves":[156],"performance":[159],"NLU,":[161],"at":[166],"time,":[169],"demonstrates":[171],"its":[172],"generalizability":[173]},"counts_by_year":[{"year":2024,"cited_by_count":8},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":11},{"year":2021,"cited_by_count":7}],"updated_date":"2026-11-03T10:50:19.383937","created_date":"2021-03-29T00:00:00"}