{"id":"https://openalex.org/W4223578248","doi":"https://doi.org/10.1109/iceic54506.2022.9748643","title":"Correlation Between Attention Heads of BERT","display_name":"Correlation Between Attention Heads of BERT","publication_year":2022,"publication_date":"2022-02-06","ids":{"openalex":"https://openalex.org/W4223578248","doi":"https://doi.org/10.1109/iceic54506.2022.9748643"},"language":"en","primary_location":{"id":"doi:10.1109/iceic54506.2022.9748643","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iceic54506.2022.9748643","pdf_url":null,"source":{"id":"https://openalex.org/S4363608213","display_name":"2022 International Conference on Electronics, Information, and Communication (ICEIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 International Conference on Electronics, Information, and Communication (ICEIC)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045330851","display_name":"Seungmo Yang","orcid":"https://orcid.org/0000-0001-6501-3812"},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":true,"raw_author_name":"Seungmo Yang","raw_affiliation_strings":["Hanyang University ERICA,Major in Bio Artificial Intelligence,Ansan,South Korea","Major in Bio Artificial Intelligence, Hanyang University ERICA, Ansan, South Korea"],"affiliations":[{"raw_affiliation_string":"Hanyang University ERICA,Major in Bio Artificial Intelligence,Ansan,South Korea","institution_ids":["https://openalex.org/I4575257"]},{"raw_affiliation_string":"Major in Bio Artificial Intelligence, Hanyang University ERICA, Ansan, South Korea","institution_ids":["https://openalex.org/I4575257"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085907258","display_name":"Mincheal Kang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mincheal Kang","raw_affiliation_strings":["Mckinsey & Co.,Japan"],"affiliations":[{"raw_affiliation_string":"Mckinsey & Co.,Japan","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100657252","display_name":"Jiwon Seo","orcid":"https://orcid.org/0000-0002-4855-5609"},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jiwon Seo","raw_affiliation_strings":["Hanyang University,Deptartment of Computer Science,Seoul,South Korea","Deptartment of Computer Science, Hanyang University, Seoul, South Korea"],"affiliations":[{"raw_affiliation_string":"Hanyang University,Deptartment of Computer Science,Seoul,South Korea","institution_ids":["https://openalex.org/I4575257"]},{"raw_affiliation_string":"Deptartment of Computer Science, Hanyang University, Seoul, South Korea","institution_ids":["https://openalex.org/I4575257"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100327650","display_name":"Younghoon Kim","orcid":"https://orcid.org/0000-0002-3049-035X"},"institutions":[{"id":"https://openalex.org/I4575257","display_name":"Hanyang University","ror":"https://ror.org/046865y68","country_code":"KR","type":"education","lineage":["https://openalex.org/I4575257"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Younghoon Kim","raw_affiliation_strings":["Hanyang University ERICA,Major in Bio Artificial Intelligence,Ansan,South Korea","Major in Bio Artificial Intelligence, Hanyang University ERICA, Ansan, South Korea"],"affiliations":[{"raw_affiliation_string":"Hanyang University ERICA,Major in Bio Artificial Intelligence,Ansan,South Korea","institution_ids":["https://openalex.org/I4575257"]},{"raw_affiliation_string":"Major in Bio Artificial Intelligence, Hanyang University ERICA, Ansan, South Korea","institution_ids":["https://openalex.org/I4575257"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5045330851"],"corresponding_institution_ids":["https://openalex.org/I4575257"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.01739927,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"3"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7566584944725037},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6793016791343689},{"id":"https://openalex.org/keywords/correlation","display_name":"Correlation","score":0.6774094104766846},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6008925437927246},{"id":"https://openalex.org/keywords/head","display_name":"Head (geology)","score":0.5065661668777466},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.4820359945297241},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.44324901700019836},{"id":"https://openalex.org/keywords/positive-correlation","display_name":"Positive correlation","score":0.4299110174179077},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4183884859085083},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.12332126498222351},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.07072854042053223},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.06610390543937683}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7566584944725037},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6793016791343689},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.6774094104766846},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6008925437927246},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.5065661668777466},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.4820359945297241},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.44324901700019836},{"id":"https://openalex.org/C2986115478","wikidata":"https://www.wikidata.org/wiki/Q186290","display_name":"Positive correlation","level":2,"score":0.4299110174179077},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4183884859085083},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.12332126498222351},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.07072854042053223},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.06610390543937683},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C126322002","wikidata":"https://www.wikidata.org/wiki/Q11180","display_name":"Internal medicine","level":1,"score":0.0},{"id":"https://openalex.org/C114793014","wikidata":"https://www.wikidata.org/wiki/Q52109","display_name":"Geomorphology","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/iceic54506.2022.9748643","is_oa":false,"landing_page_url":"https://doi.org/10.1109/iceic54506.2022.9748643","pdf_url":null,"source":{"id":"https://openalex.org/S4363608213","display_name":"2022 International Conference on Electronics, Information, and Communication (ICEIC)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2022 International Conference on Electronics, Information, and Communication (ICEIC)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8100000023841858,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G286657951","display_name":null,"funder_award_id":"2020RIGIA1011471","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":10,"referenced_works":["https://openalex.org/W1815252973","https://openalex.org/W2282821441","https://openalex.org/W2616247523","https://openalex.org/W2896457183","https://openalex.org/W2912351236","https://openalex.org/W2970120757","https://openalex.org/W3102564565","https://openalex.org/W4237723258","https://openalex.org/W4385245566","https://openalex.org/W6755207826"],"related_works":["https://openalex.org/W2364238915","https://openalex.org/W3090746915","https://openalex.org/W4313021653","https://openalex.org/W2894774984","https://openalex.org/W4378650545","https://openalex.org/W4400576964","https://openalex.org/W2393392990","https://openalex.org/W4281298071","https://openalex.org/W4308579029","https://openalex.org/W2766513281"],"abstract_inverted_index":{"Recently,":[0],"as":[1,54],"deep":[2,17],"learning":[3,18],"achieves":[4],"tremendous":[5],"success":[6],"in":[7,24,90],"a":[8,94],"variety":[9],"of":[10,28,51,58,86,93,115,125,159],"application":[11],"domains,":[12],"natural":[13],"language":[14],"processing":[15],"adopting":[16],"also":[19],"has":[20,69],"become":[21],"very":[22],"widespread":[23],"research.":[25],"The":[26],"performance":[27],"typical":[29],"such":[30,53,145],"models":[31,37],"like":[32],"Transformer,":[33],"BERT":[34,96],"and":[35,41,98,140],"GPT":[36],"is":[38],"quite":[39],"excellent":[40],"near":[42],"human":[43],"performance.":[44],"However,":[45],"due":[46],"to":[47,74,123,155],"its":[48],"complicate":[49],"structure":[50,158],"operations":[52],"self-attention,":[55],"the":[56,64,81,84,111,126,141,148,157],"role":[57],"internal":[59],"outputs":[60],"between":[61,66,83,150],"layers":[62],"or":[63],"relationship":[65],"latent":[67,113],"vectors":[68,114],"been":[70],"seldomly":[71],"studied":[72],"compared":[73],"CNNs.":[75],"In":[76],"this":[77],"work,":[78],"we":[79,108,131],"calculate":[80],"correlation":[82,139,149],"output":[85,112],"multiple":[87],"self-attention":[88],"heads":[89,136,151],"each":[91],"layer":[92],"pre-trained":[95],"model":[97],"investigate":[99],"if":[100,110],"there":[101,134],"exist":[102],"redundantly":[103],"trained":[104],"ones,":[105],"that":[106,133,144],"is,":[107],"test":[109],"an":[116],"attention":[117],"head":[118],"can":[119],"be":[120],"linearly":[121],"transformed":[122],"those":[124],"other":[127],"head.":[128],"By":[129],"experiments,":[130],"show":[132],"are":[135],"with":[137],"high":[138],"result":[142],"implies":[143],"examination":[146],"on":[147],"may":[152],"help":[153],"us":[154],"optimize":[156],"BERT.":[160]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}