{"id":"https://openalex.org/W3202284475","doi":null,"title":"MemBERT: Injecting Unstructured Knowledge into BERT","display_name":"MemBERT: Injecting Unstructured Knowledge into BERT","publication_year":2021,"publication_date":"2021-09-02","ids":{"openalex":"https://openalex.org/W3202284475","mag":"3202284475"},"language":"en","primary_location":{"id":"mag:3202284475","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2110.00125","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://export.arxiv.org/pdf/2110.00125","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058272366","display_name":"Federico Ruggeri","orcid":"https://orcid.org/0000-0002-1697-8586","observed_orcids":["https://orcid.org/0000-0002-1697-8586"]},"institutions":[{"id":"https://openalex.org/I9360294","display_name":"University of Bologna","ror":"https://ror.org/01111rn36","country_code":"IT","type":"education","lineage":["https://openalex.org/I9360294"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Federico Ruggeri","raw_affiliation_strings":["University of Bologna"],"raw_orcid":"https://orcid.org/0000-0002-1697-8586","affiliations":[{"raw_affiliation_string":"University of Bologna","institution_ids":["https://openalex.org/I9360294"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062424045","display_name":"Marco Lippi","orcid":"https://orcid.org/0000-0002-9663-1071","observed_orcids":["https://orcid.org/0000-0002-9663-1071"]},"institutions":[{"id":"https://openalex.org/I122346577","display_name":"University of Modena and Reggio Emilia","ror":"https://ror.org/02d4c4y02","country_code":"IT","type":"education","lineage":["https://openalex.org/I122346577"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Marco Lippi","raw_affiliation_strings":["Univ. of Modena and Reggio Emilia#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-9663-1071","affiliations":[{"raw_affiliation_string":"Univ. of Modena and Reggio Emilia#TAB#","institution_ids":["https://openalex.org/I122346577"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5080634543","display_name":"Paolo Torroni","orcid":"https://orcid.org/0000-0002-9253-8638","observed_orcids":["https://orcid.org/0000-0002-9253-8638","https://orcid.org/0000-0002-9294-0408"]},"institutions":[{"id":"https://openalex.org/I9360294","display_name":"University of Bologna","ror":"https://ror.org/01111rn36","country_code":"IT","type":"education","lineage":["https://openalex.org/I9360294"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Paolo Torroni","raw_affiliation_strings":["University of Bologna"],"raw_orcid":"https://orcid.org/0000-0002-9253-8638","affiliations":[{"raw_affiliation_string":"University of Bologna","institution_ids":["https://openalex.org/I9360294"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9814000129699707,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0066999997943639755,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.00419999985024333,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memory-augmented-transformer","display_name":"memory-augmented transformer","score":0.722000002861023},{"id":"https://openalex.org/keywords/knowledge-injection","display_name":"knowledge injection","score":0.9200000166893005},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.7080000042915344},{"id":"https://openalex.org/keywords/model-interpretability","display_name":"model interpretability","score":0.8059999942779541},{"id":"https://openalex.org/keywords/knowledge-acquisition-bottleneck","display_name":"knowledge acquisition bottleneck","score":0.5849999785423279}],"concepts":[{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.9413156509399414},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7743513584136963},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.737290620803833},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.7019292116165161},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6931468844413757},{"id":"https://openalex.org/C207685749","wikidata":"https://www.wikidata.org/wiki/Q2088941","display_name":"Domain knowledge","level":2,"score":0.6189077496528625},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5480626821517944},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.47751346230506897},{"id":"https://openalex.org/C2781252014","wikidata":"https://www.wikidata.org/wiki/Q1141900","display_name":"Unstructured data","level":3,"score":0.4573788344860077},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3797372877597809},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.22067520022392273},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.11152282357215881},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.09504950046539307},{"id":"https://openalex.org/C75684735","wikidata":"https://www.wikidata.org/wiki/Q858810","display_name":"Big data","level":2,"score":0.06694021821022034},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:3202284475","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2110.00125","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:3202284475","doi":null,"is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2110.00125","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":39,"referenced_works":["https://openalex.org/W1793121960","https://openalex.org/W2201581102","https://openalex.org/W2250762536","https://openalex.org/W2296073425","https://openalex.org/W2327805699","https://openalex.org/W2565304260","https://openalex.org/W2759653627","https://openalex.org/W2786715987","https://openalex.org/W2798638375","https://openalex.org/W2798838283","https://openalex.org/W2889436406","https://openalex.org/W2889855823","https://openalex.org/W2898875342","https://openalex.org/W2914287793","https://openalex.org/W2934842096","https://openalex.org/W2945976633","https://openalex.org/W2955227499","https://openalex.org/W2963341956","https://openalex.org/W2963687836","https://openalex.org/W2968210605","https://openalex.org/W2970327629","https://openalex.org/W2978017171","https://openalex.org/W2985900875","https://openalex.org/W2988841832","https://openalex.org/W2991675023","https://openalex.org/W2994673210","https://openalex.org/W2995753487","https://openalex.org/W3006087551","https://openalex.org/W3013192639","https://openalex.org/W3015468748","https://openalex.org/W3031696893","https://openalex.org/W3039578880","https://openalex.org/W3047312186","https://openalex.org/W3093260394","https://openalex.org/W3093463288","https://openalex.org/W3094083263","https://openalex.org/W3152624744","https://openalex.org/W3161206517","https://openalex.org/W3199638386"],"related_works":["https://openalex.org/W6903820829","https://openalex.org/W3126974869","https://openalex.org/W3205010600","https://openalex.org/W7028417817","https://openalex.org/W7151810879","https://openalex.org/W4360764510","https://openalex.org/W4385479968","https://openalex.org/W4368304444","https://openalex.org/W4412889752","https://openalex.org/W4385573010"],"abstract_inverted_index":{"Transformers":[0],"changed":[1],"modern":[2],"NLP":[3,66],"in":[4,26,55],"many":[5],"ways.":[6],"However,":[7],"they":[8,19],"can":[9],"hardly":[10],"exploit":[11],"domain":[12,52],"knowledge,":[13],"and":[14,75],"like":[15],"other":[16],"blackbox":[17],"models,":[18],"lack":[20],"interpretability.":[21],"Unfortunately,":[22],"structured":[23],"knowledge":[24,35,53],"injection,":[25],"the":[27],"long":[28],"run,":[29],"risks":[30],"to":[31],"suffer":[32],"from":[33],"a":[34,41],"acquisition":[36],"bottleneck.":[37],"We":[38],"thus":[39],"propose":[40],"memory":[42],"enhancement":[43],"of":[44,50],"transformer":[45],"models":[46],"that":[47,69],"makes":[48],"use":[49],"unstructured":[51],"expressed":[54],"plain":[56],"natural":[57],"language.":[58],"An":[59],"experimental":[60],"evaluation":[61],"conducted":[62],"on":[63],"two":[64],"challenging":[65],"tasks":[67],"demonstrates":[68],"our":[70],"approach":[71],"yields":[72],"better":[73],"performance":[74],"model":[76],"interpretability":[77],"than":[78],"baseline":[79],"transformer-based":[80],"architectures.":[81]},"counts_by_year":[{"year":2024,"cited_by_count":1}],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2021-10-11T00:00:00"}