{"id":"https://openalex.org/W4382812356","doi":"https://doi.org/10.3390/fi15070230","title":"Synonyms, Antonyms and Factual Knowledge in BERT Heads","display_name":"Synonyms, Antonyms and Factual Knowledge in BERT Heads","publication_year":2023,"publication_date":"2023-06-29","ids":{"openalex":"https://openalex.org/W4382812356","doi":"https://doi.org/10.3390/fi15070230"},"language":"en","primary_location":{"id":"doi:10.3390/fi15070230","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi15070230","pdf_url":"https://www.mdpi.com/1999-5903/15/7/230/pdf?version=1688024294","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.mdpi.com/1999-5903/15/7/230/pdf?version=1688024294","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092378227","display_name":"Lorenzo Serina","orcid":"https://orcid.org/0000-0002-1835-0442"},"institutions":[{"id":"https://openalex.org/I79940851","display_name":"University of Brescia","ror":"https://ror.org/02q2d2610","country_code":"IT","type":"education","lineage":["https://openalex.org/I79940851"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Lorenzo Serina","raw_affiliation_strings":["Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy"],"raw_orcid":"https://orcid.org/0000-0002-1835-0442","affiliations":[{"raw_affiliation_string":"Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy","institution_ids":["https://openalex.org/I79940851"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5044037409","display_name":"Luca Putelli","orcid":"https://orcid.org/0009-0008-5055-6812"},"institutions":[{"id":"https://openalex.org/I79940851","display_name":"University of Brescia","ror":"https://ror.org/02q2d2610","country_code":"IT","type":"education","lineage":["https://openalex.org/I79940851"]}],"countries":["IT"],"is_corresponding":true,"raw_author_name":"Luca Putelli","raw_affiliation_strings":["Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy","institution_ids":["https://openalex.org/I79940851"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073627176","display_name":"Alfonso Gerevini","orcid":"https://orcid.org/0000-0001-9008-6386"},"institutions":[{"id":"https://openalex.org/I79940851","display_name":"University of Brescia","ror":"https://ror.org/02q2d2610","country_code":"IT","type":"education","lineage":["https://openalex.org/I79940851"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Alfonso Gerevini","raw_affiliation_strings":["Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy","institution_ids":["https://openalex.org/I79940851"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068039203","display_name":"Ivan Serina","orcid":"https://orcid.org/0000-0002-7785-9492"},"institutions":[{"id":"https://openalex.org/I79940851","display_name":"University of Brescia","ror":"https://ror.org/02q2d2610","country_code":"IT","type":"education","lineage":["https://openalex.org/I79940851"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Ivan Serina","raw_affiliation_strings":["Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy"],"raw_orcid":"https://orcid.org/0000-0002-7785-9492","affiliations":[{"raw_affiliation_string":"Department of Information Engineering, Universit\u00e0 Degli Studi di Brescia, Via Branze 38, 25100 Brescia, Italy","institution_ids":["https://openalex.org/I79940851"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5044037409","https://openalex.org/A5092378227"],"corresponding_institution_ids":["https://openalex.org/I79940851"],"apc_list":{"value":1400,"currency":"CHF","value_usd":1515},"apc_paid":{"value":1400,"currency":"CHF","value_usd":1515},"fwci":0.994,"has_fulltext":true,"cited_by_count":6,"citation_normalized_percentile":{"value":0.80406455,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"15","issue":"7","first_page":"230","last_page":"230"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9991999864578247,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11710","display_name":"Biomedical Text Mining and Ontologies","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/1312","display_name":"Molecular Biology"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.9965000152587891,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8927061557769775},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7315918803215027},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.668728768825531},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.5637682676315308},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5206900835037231},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4962924122810364},{"id":"https://openalex.org/keywords/cluster-analysis","display_name":"Cluster analysis","score":0.4805788993835449},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.47534629702568054},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.4373226463794708},{"id":"https://openalex.org/keywords/information-retrieval","display_name":"Information retrieval","score":0.3572503626346588},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.3298186659812927}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8927061557769775},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7315918803215027},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.668728768825531},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.5637682676315308},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5206900835037231},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4962924122810364},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.4805788993835449},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.47534629702568054},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4373226463794708},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.3572503626346588},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.3298186659812927},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.3390/fi15070230","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi15070230","pdf_url":"https://www.mdpi.com/1999-5903/15/7/230/pdf?version=1688024294","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},{"id":"pmh:oai:iris.unibs.it:11379/582226","is_oa":true,"landing_page_url":"https://hdl.handle.net/11379/582226","pdf_url":"https://iris.unibs.it/bitstream/11379/582226/2/futureinternet-15-00230.pdf","source":{"id":"https://openalex.org/S4306400804","display_name":"Institutional Research Information System (Universit\u00e0 degli Studi di Brescia)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I66752286","host_organization_name":"University of Milano-Bicocca","host_organization_lineage":["https://openalex.org/I66752286"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/article"},{"id":"pmh:oai:RePEc:gam:jftint:v:15:y:2023:i:7:p:230-:d:1182617","is_oa":false,"landing_page_url":"https://www.mdpi.com/1999-5903/15/7/230/","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"pmh:oai:doaj.org/article:f6c3e41dbc09459ba652b1c776b32117","is_oa":true,"landing_page_url":"https://doaj.org/article/f6c3e41dbc09459ba652b1c776b32117","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Future Internet, Vol 15, Iss 7, p 230 (2023)","raw_type":"article"},{"id":"pmh:oai:mdpi.com:/1999-5903/15/7/230/","is_oa":true,"landing_page_url":"https://dx.doi.org/10.3390/fi15070230","pdf_url":null,"source":{"id":"https://openalex.org/S4306400947","display_name":"MDPI (MDPI AG)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210097602","host_organization_name":"Multidisciplinary Digital Publishing Institute (Switzerland)","host_organization_lineage":["https://openalex.org/I4210097602"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Future Internet","raw_type":"Text"}],"best_oa_location":{"id":"doi:10.3390/fi15070230","is_oa":true,"landing_page_url":"https://doi.org/10.3390/fi15070230","pdf_url":"https://www.mdpi.com/1999-5903/15/7/230/pdf?version=1688024294","source":{"id":"https://openalex.org/S34838331","display_name":"Future Internet","issn_l":"1999-5903","issn":["1999-5903"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310310987","host_organization_name":"Multidisciplinary Digital Publishing Institute","host_organization_lineage":["https://openalex.org/P4310310987"],"host_organization_lineage_names":["Multidisciplinary Digital Publishing Institute"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Future Internet","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4382812356.pdf"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W2067191022","https://openalex.org/W2100961932","https://openalex.org/W2785611959","https://openalex.org/W2896457183","https://openalex.org/W2948947170","https://openalex.org/W2950784811","https://openalex.org/W2970120757","https://openalex.org/W2971258845","https://openalex.org/W2972324944","https://openalex.org/W3021533447","https://openalex.org/W3044438666","https://openalex.org/W3090686793","https://openalex.org/W3118017018","https://openalex.org/W3118485687","https://openalex.org/W3152884768","https://openalex.org/W3156170450","https://openalex.org/W3167282650","https://openalex.org/W3202712981","https://openalex.org/W4214812012","https://openalex.org/W4306317657","https://openalex.org/W4323903888","https://openalex.org/W6755207826","https://openalex.org/W6762537594","https://openalex.org/W6763121668","https://openalex.org/W6763367618"],"related_works":["https://openalex.org/W17155033","https://openalex.org/W3207760230","https://openalex.org/W1496222301","https://openalex.org/W1590307681","https://openalex.org/W2536018345","https://openalex.org/W4312814274","https://openalex.org/W4285370786","https://openalex.org/W2296488620","https://openalex.org/W2358353312","https://openalex.org/W2353836703"],"abstract_inverted_index":{"In":[0,56,104],"recent":[1],"years,":[2],"many":[3],"studies":[4],"have":[5],"been":[6],"devoted":[7],"to":[8,22,63],"discovering":[9],"the":[10,50,53,95],"inner":[11],"workings":[12],"of":[13,44,49,120],"Transformer-based":[14],"models,":[15],"such":[16,125],"as":[17,126],"BERT,":[18],"for":[19,93],"instance,":[20],"attempting":[21],"identify":[23,64],"what":[24],"information":[25,40],"is":[26,32],"contained":[27],"within":[28],"them.":[29],"However,":[30],"little":[31],"known":[33],"about":[34],"how":[35,111],"these":[36],"models":[37],"store":[38],"this":[39,57],"in":[41,76,101],"their":[42],"millions":[43],"parameters":[45],"and":[46,72,87,117,128],"which":[47],"parts":[48],"architecture":[51],"are":[52],"most":[54,96],"important.":[55],"work,":[58],"we":[59,108],"propose":[60],"an":[61],"approach":[62,79],"self-attention":[65],"mechanisms,":[66],"called":[67],"heads,":[68],"that":[69],"contain":[70],"semantic":[71],"real-world":[73],"factual":[74,121],"knowledge":[75,122],"BERT.":[77],"Our":[78],"includes":[80],"a":[81,89,102],"metric":[82],"computed":[83],"from":[84],"attention":[85],"weights":[86],"exploits":[88],"standard":[90],"clustering":[91],"algorithm":[92],"extracting":[94],"relevant":[97],"connections":[98],"between":[99],"tokens":[100],"head.":[103],"our":[105],"experimental":[106],"analysis,":[107],"focus":[109],"on":[110],"heads":[112],"can":[113],"connect":[114],"synonyms,":[115],"antonyms":[116],"several":[118],"types":[119],"regarding":[123],"subjects":[124],"geography":[127],"medicine.":[129]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2023-07-02T00:00:00"}