{"id":"https://openalex.org/W3033187248","doi":"https://doi.org/10.48550/arxiv.2006.03654","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","display_name":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","publication_year":2020,"publication_date":"2020-06-05","ids":{"openalex":"https://openalex.org/W3033187248","doi":"https://doi.org/10.48550/arxiv.2006.03654","mag":"3033187248"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2006.03654","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.03654","pdf_url":"https://arxiv.org/pdf/2006.03654","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2006.03654","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019259019","display_name":"Pengcheng He","orcid":"https://orcid.org/0000-0002-7860-503X","observed_orcids":["https://orcid.org/0000-0002-7860-503X"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"He, Pengcheng","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100374810","display_name":"Xiaodong Liu","orcid":"https://orcid.org/0009-0004-6648-2302","observed_orcids":["https://orcid.org/0009-0004-6648-2302"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Xiaodong","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114910293","display_name":"Jianfeng Gao","orcid":"https://orcid.org/0000-0002-5702-6143","observed_orcids":["https://orcid.org/0000-0002-5702-6143"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gao, Jianfeng","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051745436","display_name":"Weizhu Chen","orcid":"https://orcid.org/0000-0003-4196-7229","observed_orcids":["https://orcid.org/0000-0003-4196-7229"]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Weizhu","raw_affiliation_strings":["Microsoft Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1290206253"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1224,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.003100000089034438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/deberta","display_name":"DeBERTa","score":0.9940000176429749},{"id":"https://openalex.org/keywords/disentangled-attention","display_name":"disentangled attention","score":0.9559999704360962},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.8230000138282776},{"id":"https://openalex.org/keywords/roberta","display_name":"RoBERTa","score":0.9900000095367432},{"id":"https://openalex.org/keywords/pre-trained-language-models","display_name":"pre-trained language models","score":0.7730000019073486},{"id":"https://openalex.org/keywords/mask-decoder","display_name":"mask decoder","score":0.6990000009536743},{"id":"https://openalex.org/keywords/virtual-adversarial-training","display_name":"virtual adversarial training","score":0.8870000243186951},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"natural language understanding","score":0.7540000081062317},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"natural language generation","score":0.9980000257492065},{"id":"https://openalex.org/keywords/superglue","display_name":"SuperGLUE","score":0.8059999942779541},{"id":"https://openalex.org/keywords/mnli","display_name":"MNLI","score":0.4320000112056732},{"id":"https://openalex.org/keywords/squad","display_name":"SQuAD","score":0.8450000286102295},{"id":"https://openalex.org/keywords/race","display_name":"race","score":0.7609999775886536},{"id":"https://openalex.org/keywords/ensemble-learning","display_name":"ensemble learning","score":0.32199999690055847}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7613390684127808},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.7198348045349121},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6315516829490662},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6251187920570374},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.598726749420166},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.5251205563545227},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.4962690472602844},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.47615882754325867},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.4695713520050049},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.45316898822784424},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4529881775379181},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4457639753818512},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.35949480533599854},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.31445151567459106},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.1733016073703766},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.14046627283096313},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1061643660068512},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C178790620","wikidata":"https://www.wikidata.org/wiki/Q11351","display_name":"Organic chemistry","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2006.03654","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.03654","pdf_url":"https://arxiv.org/pdf/2006.03654","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2006.03654","doi":"https://doi.org/10.48550/arxiv.2006.03654","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2006.03654","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2006.03654","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.03654","pdf_url":"https://arxiv.org/pdf/2006.03654","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3033187248.pdf","grobid_xml":"https://content.openalex.org/works/W3033187248.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1522301498","https://openalex.org/W1566289585","https://openalex.org/W1599016936","https://openalex.org/W2013494846","https://openalex.org/W2130158090","https://openalex.org/W2145755360","https://openalex.org/W2251939518","https://openalex.org/W2396767181","https://openalex.org/W2525127255","https://openalex.org/W2525332836","https://openalex.org/W2606964149","https://openalex.org/W2607892599","https://openalex.org/W2626778328","https://openalex.org/W2739351760","https://openalex.org/W2768282280","https://openalex.org/W2789541106","https://openalex.org/W2804897457","https://openalex.org/W2805206884","https://openalex.org/W2806120502","https://openalex.org/W2886424491","https://openalex.org/W2898662126","https://openalex.org/W2919624000","https://openalex.org/W2938830017","https://openalex.org/W2940744433","https://openalex.org/W2945260553","https://openalex.org/W2946659172","https://openalex.org/W2950813464","https://openalex.org/W2962753370","https://openalex.org/W2963310665","https://openalex.org/W2963323070","https://openalex.org/W2963341956","https://openalex.org/W2963748441","https://openalex.org/W2963854351","https://openalex.org/W2964110616","https://openalex.org/W2964159205","https://openalex.org/W2965373594","https://openalex.org/W2966892770","https://openalex.org/W2966989210","https://openalex.org/W2969670656","https://openalex.org/W2973727699","https://openalex.org/W2975059944","https://openalex.org/W2981037730","https://openalex.org/W2985884876","https://openalex.org/W2989033444","https://openalex.org/W2990704537","https://openalex.org/W2994673210","https://openalex.org/W2994689640","https://openalex.org/W2996035354","https://openalex.org/W3000514857","https://openalex.org/W3015468748","https://openalex.org/W3017003177","https://openalex.org/W3026404337","https://openalex.org/W3030163527","https://openalex.org/W3035576622","https://openalex.org/W3082274269","https://openalex.org/W3105082862","https://openalex.org/W3119866685","https://openalex.org/W3128464707","https://openalex.org/W3131870090","https://openalex.org/W3156333129"],"related_works":["https://openalex.org/W3211686893","https://openalex.org/W4391376505","https://openalex.org/W4411851928","https://openalex.org/W3016473712","https://openalex.org/W4221158424","https://openalex.org/W2950813464","https://openalex.org/W2966892770","https://openalex.org/W4287265459","https://openalex.org/W4224911293","https://openalex.org/W4224088803"],"abstract_inverted_index":{"Recent":[0],"progress":[1],"in":[2,96,105,244],"pre-trained":[3],"neural":[4],"language":[5,15,142],"models":[6,40],"has":[7],"significantly":[8,129],"improved":[9],"the":[10,36,48,68,97,102,131,137,162,224,229,233,241,253,259,269],"performance":[11,138,221,231],"of":[12,133,139,161,172,211,246,263],"many":[13],"natural":[14,141,146],"processing":[16],"(NLP)":[17],"tasks.":[18,151],"In":[19,108],"this":[20],"paper":[21],"we":[22,200],"propose":[23],"a":[24,110,155,169,206,273],"new":[25,111],"model":[26,106,134,157,227,256],"architecture":[27],"DeBERTa":[28,156,203,226,255],"(Decoding-enhanced":[29],"BERT":[30,37],"with":[31,215],"disentangled":[32,49,76],"attention)":[33],"that":[34,60,126,209],"improves":[35],"and":[38,64,67,81,136,145,192,252],"RoBERTa":[39],"using":[41,57,75],"two":[42,58],"novel":[43],"techniques.":[44],"The":[45,219],"first":[46,242],"is":[47,55,90,116],"attention":[50,69],"mechanism,":[51],"where":[52],"each":[53],"word":[54],"represented":[56],"vectors":[59],"encode":[61],"its":[62],"content":[63],"position,":[65],"respectively,":[66],"weights":[70],"among":[71],"words":[72],"are":[73],"computed":[74],"matrices":[77],"on":[78,159,168,177,184,232],"their":[79],"contents":[80],"relative":[82],"positions,":[83],"respectively.":[84],"Second,":[85],"an":[86],"enhanced":[87],"mask":[88],"decoder":[89],"used":[91,117],"to":[92,100,120,153],"incorporate":[93],"absolute":[94],"positions":[95],"decoding":[98],"layer":[99],"predict":[101],"masked":[103],"tokens":[104],"pre-training.":[107],"addition,":[109],"virtual":[112],"adversarial":[113],"training":[114,163,205],"method":[115],"for":[118,240],"fine-tuning":[119],"improve":[121,130],"models'":[122],"generalization.":[123],"We":[124],"show":[125],"these":[127],"techniques":[128],"efficiency":[132],"pre-training":[135],"both":[140],"understanding":[143],"(NLU)":[144],"langauge":[147],"generation":[148],"(NLG)":[149],"downstream":[150],"Compared":[152],"RoBERTa-Large,":[154],"trained":[158],"half":[160],"data":[164],"performs":[165],"consistently":[166],"better":[167],"wide":[170],"range":[171],"NLP":[173],"tasks,":[174],"achieving":[175],"improvements":[176],"MNLI":[178],"by":[179,187,194,204,272],"+0.9%":[180],"(90.2%":[181],"vs.":[182,190,197],"91.1%),":[183],"SQuAD":[185],"v2.0":[186],"+2.3%":[188],"(88.4%":[189],"90.7%)":[191],"RACE":[193],"+3.6%":[195],"(83.2%":[196],"86.8%).":[198],"Notably,":[199],"scale":[201],"up":[202],"larger":[207],"version":[208],"consists":[210],"48":[212],"Transform":[213],"layers":[214],"1.5":[216],"billion":[217],"parameters.":[218],"significant":[220],"boost":[222],"makes":[223],"single":[225],"surpass":[228],"human":[230,270],"SuperGLUE":[234,260],"benchmark":[235],"(Wang":[236],"et":[237],"al.,":[238],"2019a)":[239],"time":[243],"terms":[245],"macro-average":[247],"score":[248],"(89.9":[249],"versus":[250,277],"89.8),":[251],"ensemble":[254],"sits":[257],"atop":[258],"leaderboard":[261],"as":[262],"January":[264],"6,":[265],"2021,":[266],"out":[267],"performing":[268],"baseline":[271],"decent":[274],"margin":[275],"(90.3":[276],"89.8).":[278]},"counts_by_year":[{"year":2026,"cited_by_count":277},{"year":2025,"cited_by_count":287},{"year":2024,"cited_by_count":249},{"year":2023,"cited_by_count":178},{"year":2022,"cited_by_count":121},{"year":2021,"cited_by_count":105},{"year":2020,"cited_by_count":7}],"updated_date":"2026-11-06T12:05:46.655746","created_date":"2020-06-12T00:00:00"}