{"id":"https://openalex.org/W4389891246","doi":"https://doi.org/10.1109/tkde.2023.3341917","title":"E2S2: Encoding-Enhanced Sequence-to-Sequence Pretraining for Language Understanding and Generation","display_name":"E2S2: Encoding-Enhanced Sequence-to-Sequence Pretraining for Language Understanding and Generation","publication_year":2023,"publication_date":"2023-12-18","ids":{"openalex":"https://openalex.org/W4389891246","doi":"https://doi.org/10.1109/tkde.2023.3341917"},"language":"en","primary_location":{"id":"doi:10.1109/tkde.2023.3341917","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2023.3341917","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053183023","display_name":"Qihuang Zhong","orcid":"https://orcid.org/0009-0001-0118-5217"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qihuang Zhong","raw_affiliation_strings":["School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0001-0118-5217","affiliations":[{"raw_affiliation_string":"School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100456723","display_name":"Liang Ding","orcid":"https://orcid.org/0000-0001-8976-2084"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Liang Ding","raw_affiliation_strings":["School of Computer Science, Faculty of Engineering, The University of Sydney, Sydney, Australia"],"raw_orcid":"https://orcid.org/0000-0001-8976-2084","affiliations":[{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, The University of Sydney, Sydney, Australia","institution_ids":["https://openalex.org/I129604602"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026501335","display_name":"Juhua Liu","orcid":"https://orcid.org/0000-0002-3907-8820"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Juhua Liu","raw_affiliation_strings":["School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-3907-8820","affiliations":[{"raw_affiliation_string":"School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060042752","display_name":"Bo Du","orcid":"https://orcid.org/0000-0002-0059-8458"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Du","raw_affiliation_strings":["School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-0059-8458","affiliations":[{"raw_affiliation_string":"School of Computer Science, National Engineering Research Center for Multimedia Software, Institute of Artificial Intelligence, and Hubei Key Laboratory of Multimedia and Network Communication Engineering, Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074103823","display_name":"Dacheng Tao","orcid":"https://orcid.org/0000-0001-7225-5449"},"institutions":[{"id":"https://openalex.org/I129604602","display_name":"The University of Sydney","ror":"https://ror.org/0384j8v12","country_code":"AU","type":"education","lineage":["https://openalex.org/I129604602"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Dacheng Tao","raw_affiliation_strings":["School of Computer Science, Faculty of Engineering, The University of Sydney, Sydney, Australia"],"raw_orcid":"https://orcid.org/0000-0001-7225-5449","affiliations":[{"raw_affiliation_string":"School of Computer Science, Faculty of Engineering, The University of Sydney, Sydney, Australia","institution_ids":["https://openalex.org/I129604602"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.0363,"has_fulltext":false,"cited_by_count":25,"citation_normalized_percentile":{"value":0.93040897,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"36","issue":"12","first_page":"8037","last_page":"8050"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9890999794006348,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8408573865890503},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.7364436984062195},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.6174004673957825},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.614093005657196},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.6032723784446716},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5675784945487976},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5465514063835144},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5406280755996704},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.49720242619514465},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.42251765727996826},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.39948034286499023},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.3962271809577942}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8408573865890503},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.7364436984062195},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.6174004673957825},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.614093005657196},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.6032723784446716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5675784945487976},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5465514063835144},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5406280755996704},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.49720242619514465},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.42251765727996826},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.39948034286499023},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.3962271809577942},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tkde.2023.3341917","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tkde.2023.3341917","pdf_url":null,"source":{"id":"https://openalex.org/S30698027","display_name":"IEEE Transactions on Knowledge and Data Engineering","issn_l":"1041-4347","issn":["1041-4347","1558-2191","2326-3865"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Knowledge and Data Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8199999928474426,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1836220556","display_name":null,"funder_award_id":"62225113","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G3648843532","display_name":null,"funder_award_id":"U23B2048","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8527334069","display_name":null,"funder_award_id":"62076186","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":72,"referenced_works":["https://openalex.org/W2098297786","https://openalex.org/W2101105183","https://openalex.org/W2162245945","https://openalex.org/W2555428947","https://openalex.org/W2606974598","https://openalex.org/W2769041395","https://openalex.org/W2785047343","https://openalex.org/W2888482885","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2933138175","https://openalex.org/W2951583236","https://openalex.org/W2959716049","https://openalex.org/W2963641307","https://openalex.org/W2963825865","https://openalex.org/W2963854351","https://openalex.org/W2964213933","https://openalex.org/W2965373594","https://openalex.org/W2970419734","https://openalex.org/W2970972587","https://openalex.org/W2997200074","https://openalex.org/W2998230451","https://openalex.org/W3001434439","https://openalex.org/W3010773219","https://openalex.org/W3011411500","https://openalex.org/W3011594683","https://openalex.org/W3034999214","https://openalex.org/W3035390927","https://openalex.org/W3035725276","https://openalex.org/W3036120435","https://openalex.org/W3098267758","https://openalex.org/W3100439847","https://openalex.org/W3105604018","https://openalex.org/W3105816068","https://openalex.org/W3119821914","https://openalex.org/W3134210100","https://openalex.org/W3153427360","https://openalex.org/W3156636935","https://openalex.org/W3167810126","https://openalex.org/W3172021172","https://openalex.org/W3173367141","https://openalex.org/W3175362188","https://openalex.org/W3185341429","https://openalex.org/W3197120431","https://openalex.org/W3200681609","https://openalex.org/W3201490875","https://openalex.org/W3204453541","https://openalex.org/W3212893438","https://openalex.org/W4205991051","https://openalex.org/W4285251104","https://openalex.org/W4288089799","https://openalex.org/W4385570901","https://openalex.org/W4385572498","https://openalex.org/W4385573170","https://openalex.org/W4385573252","https://openalex.org/W6622388105","https://openalex.org/W6632455782","https://openalex.org/W6679436768","https://openalex.org/W6682421076","https://openalex.org/W6745297980","https://openalex.org/W6748452836","https://openalex.org/W6762122294","https://openalex.org/W6762521896","https://openalex.org/W6763701032","https://openalex.org/W6766673545","https://openalex.org/W6769627184","https://openalex.org/W6771917389","https://openalex.org/W6773829392","https://openalex.org/W6778883912","https://openalex.org/W6779068807","https://openalex.org/W6779791889","https://openalex.org/W6789500345"],"related_works":["https://openalex.org/W4283822356","https://openalex.org/W1950940422","https://openalex.org/W2129146436","https://openalex.org/W2032507829","https://openalex.org/W2147282173","https://openalex.org/W382594479","https://openalex.org/W2152921782","https://openalex.org/W2470045054","https://openalex.org/W3034878914","https://openalex.org/W2575772232"],"abstract_inverted_index":{"Sequence-to-sequence":[0],"(seq2seq)":[1],"learning":[2,134],"is":[3],"a":[4,176],"popular":[5],"fashion":[6],"for":[7],"large-scale":[8],"pretraining":[9,16,89],"language":[10,60,182,214,256],"models.":[11],"However,":[12],"the":[13,24,29,50,53,65,74,77,98,108,117,126,141,146,151,164,172,190,212,237],"previous":[14],"seq2seq":[15,58,88,99,167,255],"models":[17,100],"generally":[18],"focus":[19],"on":[20,23,116,211,227,254],"reconstructive":[21],"objectives":[22,115],"decoder":[25,56,75],"side":[26,119],"and":[27,55,62,80,131,154,159,184,199,219],"neglect":[28],"effect":[30],"of":[31,52,143,166,179,192],"encoder-side":[32],"supervision,":[33],"which":[34,96],"we":[35,46,84,206],"argue":[36],"may":[37],"lead":[38],"to":[39,169,235],"sub-optimal":[40],"performance.":[41],"To":[42],"verify":[43],"our":[44,247],"hypothesis,":[45],"first":[47],"empirically":[48],"study":[49],"functionalities":[51],"encoder":[54,66,118,147],"in":[57],"pretrained":[59],"models,":[61,196],"find":[63],"that":[64,246],"takes":[67],"an":[68,86],"important":[69],"but":[70],"under-exploitation":[71],"role":[72],"than":[73],"regarding":[76],"downstream":[78,180],"performance":[79,191],"neuron":[81],"activation.":[82],"Therefore,":[83],"propose":[85],"encoding-enhanced":[87],"strategy,":[90],"namely":[91],"<sc":[92],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[93,222],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">E2S2</small>":[94],",":[95],"improves":[97,189],"via":[101],"integrating":[102],"more":[103],"efficient":[104],"self-supervised":[105,114],"information":[106],"into":[107],"encoders.":[109],"Specifically,":[110],"E2S2":[111,187],"adopts":[112],"two":[113,121],"from":[120,240],"aspects:":[122],"1)":[123],"locally":[124],"denoising":[125],"corrupted":[127],"sentence":[128,136],"(denoising":[129],"objective);":[130],"2)":[132],"globally":[133],"better":[135,241],"representations":[137],"(contrastive":[138],"objective).":[139],"With":[140],"help":[142],"both":[144],"objectives,":[145],"can":[148],"effectively":[149],"distinguish":[150],"noise":[152],"tokens":[153],"capture":[155],"high-level":[156],"(i.e.,":[157],"syntactic":[158],"semantic)":[160],"knowledge,":[161],"thus":[162],"strengthening":[163],"ability":[165],"model":[168,257],"accurately":[170],"achieve":[171,207],"conditional":[173],"generation.":[174],"On":[175],"large":[177],"diversity":[178],"natural":[181],"understanding":[183,215],"generation":[185],"tasks,":[186],"dominantly":[188],"its":[193],"powerful":[194],"backbone":[195],"e.g.,":[197],"BART":[198,204],"T5.":[200],"For":[201],"example,":[202],"upon":[203],"backbone,":[205],"+1.1%":[208],"averaged":[209],"gain":[210],"general":[213],"evaluation":[216],"(GLUE)":[217],"benchmark":[218],"+1.75%":[220],"<inline-formula":[221],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><tex-math":[223],"notation=\"LaTeX\">$F_{0.5}$</tex-math></inline-formula>":[224],"score":[225],"improvement":[226,238],"CoNLL2014":[228],"dataset.":[229],"We":[230,244],"also":[231],"provide":[232],"in-depth":[233],"analyses":[234],"show":[236],"stems":[239],"linguistic":[242],"representation.":[243],"hope":[245],"work":[248],"will":[249],"foster":[250],"future":[251],"self-supervision":[252],"research":[253],"pretraining.":[258]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":6},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":2}],"updated_date":"2026-07-21T08:15:58.654021","created_date":"2023-12-19T00:00:00"}