{"id":"https://openalex.org/W3208000343","doi":"https://doi.org/10.1109/access.2021.3122273","title":"Improving BERT With Self-Supervised Attention","display_name":"Improving BERT With Self-Supervised Attention","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3208000343","doi":"https://doi.org/10.1109/access.2021.3122273","mag":"3208000343"},"language":"en","primary_location":{"id":"doi:10.1109/access.2021.3122273","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3122273","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1109/access.2021.3122273","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065737531","display_name":"Yiren Chen","orcid":"https://orcid.org/0000-0001-5608-5144"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yiren Chen","raw_affiliation_strings":["Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5608-5144","affiliations":[{"raw_affiliation_string":"Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074506535","display_name":"Xiaoyu Kou","orcid":"https://orcid.org/0000-0002-1238-967X"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaoyu Kou","raw_affiliation_strings":["Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-1238-967X","affiliations":[{"raw_affiliation_string":"Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5018600735","display_name":"Jiangang Bai","orcid":"https://orcid.org/0000-0002-2299-2464"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiangang Bai","raw_affiliation_strings":["Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2299-2464","affiliations":[{"raw_affiliation_string":"Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024097240","display_name":"Yunhai Tong","orcid":"https://orcid.org/0000-0001-8735-2516"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yunhai Tong","raw_affiliation_strings":["Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8735-2516","affiliations":[{"raw_affiliation_string":"Key Laboratory of Machine Perception (MOE), School of Electronics Engineering and Computer Science, Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5065737531"],"corresponding_institution_ids":["https://openalex.org/I20231570"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":0.6996,"has_fulltext":false,"cited_by_count":8,"citation_normalized_percentile":{"value":0.76869469,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"9","issue":null,"first_page":"144129","last_page":"144139"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8778473138809204},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.6636606454849243},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6528801321983337},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6159369349479675},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.6094156503677368},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5303282141685486},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.49783754348754883}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8778473138809204},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6636606454849243},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6528801321983337},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6159369349479675},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.6094156503677368},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5303282141685486},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.49783754348754883},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1109/access.2021.3122273","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3122273","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2004.03808","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.03808","pdf_url":"https://arxiv.org/pdf/2004.03808","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:doaj.org/article:94a8882ea21c47ea88a5ffc6bb654529","is_oa":true,"landing_page_url":"https://doaj.org/article/94a8882ea21c47ea88a5ffc6bb654529","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 9, Pp 144129-144139 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2021.3122273","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2021.3122273","pdf_url":null,"source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6399999856948853}],"awards":[{"id":"https://openalex.org/G644216735","display_name":null,"funder_award_id":"2020YFB2103402","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":54,"referenced_works":["https://openalex.org/W131533222","https://openalex.org/W1599016936","https://openalex.org/W2130158090","https://openalex.org/W2251294039","https://openalex.org/W2251939518","https://openalex.org/W2282821441","https://openalex.org/W2465978385","https://openalex.org/W2470673105","https://openalex.org/W2516809705","https://openalex.org/W2536725458","https://openalex.org/W2597655663","https://openalex.org/W2739351760","https://openalex.org/W2741375528","https://openalex.org/W2828845773","https://openalex.org/W2888882903","https://openalex.org/W2896457183","https://openalex.org/W2898700502","https://openalex.org/W2905266130","https://openalex.org/W2923014074","https://openalex.org/W2923978210","https://openalex.org/W2952620744","https://openalex.org/W2954278700","https://openalex.org/W2962369866","https://openalex.org/W2962858109","https://openalex.org/W2963310665","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2963748441","https://openalex.org/W2963846996","https://openalex.org/W2963854351","https://openalex.org/W2964024811","https://openalex.org/W2965373594","https://openalex.org/W2971296908","https://openalex.org/W2978670439","https://openalex.org/W3104033643","https://openalex.org/W3106031848","https://openalex.org/W4288374659","https://openalex.org/W4313908941","https://openalex.org/W4322614701","https://openalex.org/W4385245566","https://openalex.org/W6605323724","https://openalex.org/W6635469476","https://openalex.org/W6679344019","https://openalex.org/W6691459498","https://openalex.org/W6728469146","https://openalex.org/W6735377749","https://openalex.org/W6739901393","https://openalex.org/W6753507521","https://openalex.org/W6755207826","https://openalex.org/W6755862295","https://openalex.org/W6760568010","https://openalex.org/W6761435113","https://openalex.org/W6765939562","https://openalex.org/W6766673545"],"related_works":["https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700","https://openalex.org/W2997152889","https://openalex.org/W2032233321","https://openalex.org/W3121970507"],"abstract_inverted_index":{"One":[0],"of":[1,6,58,103,121],"the":[2,28,45,56,59,92,96],"most":[3],"popular":[4],"paradigms":[5],"applying":[7],"large":[8],"pre-trained":[9],"NLP":[10],"models":[11],"such":[12],"as":[13,27,39],"BERT":[14,61,107,132],"is":[15],"to":[16,50,75,113],"fine-tune":[17],"it":[18],"on":[19,33],"a":[20,68,110,119],"smaller":[21,34],"dataset.":[22],"However,":[23],"one":[24],"challenge":[25],"remains":[26],"fine-tuned":[29,60,93],"model":[30,94],"often":[31],"overfits":[32],"datasets.":[35],"This":[36],"problem":[37],"manifests":[38],"irrelevant":[40],"or":[41],"misleading":[42],"words":[43],"in":[44],"sentences,":[46],"which":[47],"are":[48],"obvious":[49],"humans,":[51],"but":[52],"can":[53],"substantially":[54],"degrade":[55],"performance":[57,127],"models.":[62],"In":[63],"this":[64,78],"paper,":[65],"we":[66,124],"propose":[67,109],"novel":[69],"technique,":[70],"called":[71],"Self-Supervised":[72],"Attention":[73],"(SSA)":[74],"help":[76],"facilitate":[77],"generalization":[79],"challenge.":[80],"Specifically,":[81],"SSA":[82,105],"automatically":[83],"generates":[84],"weak,":[85],"token-level":[86],"attention":[87],"labels":[88],"iteratively":[89],"by":[90],"probing":[91],"from":[95],"previous":[97],"iteration.We":[98],"investigate":[99],"two":[100],"different":[101],"ways":[102],"integrating":[104],"into":[106],"and":[108],"hybrid":[111],"approach":[112],"combine":[114],"their":[115],"benefits.":[116],"Empirically,":[117],"through":[118],"variety":[120],"public":[122],"datasets,":[123],"illustrate":[125],"significant":[126],"improvement":[128],"using":[129],"our":[130],"SSA-enhanced":[131],"model.":[133]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-05-06T08:25:59.206177","created_date":"2025-10-10T00:00:00"}