{"id":"https://openalex.org/W4389520380","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.751","title":"Dissecting Recall of Factual Associations in Auto-Regressive Language Models","display_name":"Dissecting Recall of Factual Associations in Auto-Regressive Language Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389520380","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.751"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.emnlp-main.751","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.751","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.751","pdf_url":"https://aclanthology.org/2023.emnlp-main.751.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2023.emnlp-main.751.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065717258","display_name":"Mor Geva","orcid":"https://orcid.org/0000-0001-9529-6315","observed_orcids":["https://orcid.org/0000-0001-9529-6315"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mor Geva","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077391471","display_name":"Jasmijn Bastings","orcid":"https://orcid.org/0000-0002-5445-4417","observed_orcids":["https://orcid.org/0000-0002-5445-4417"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jasmijn Bastings","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5037657908","display_name":"Katja Filippova","orcid":"https://orcid.org/0009-0007-5308-0904","observed_orcids":["https://orcid.org/0009-0007-5308-0904"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Katja Filippova","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5047817959","display_name":"Amir Globerson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]},{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Amir Globerson","raw_affiliation_strings":["Google Research","Tel Aviv University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google Research","institution_ids":["https://openalex.org/I1291425158"]},{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":34.7955,"has_fulltext":true,"cited_by_count":307,"citation_normalized_percentile":{"value":0.99800835,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"12216","last_page":"12235"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9667999744415283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9667999744415283,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.010900000110268593,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.009399999864399433,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/factual-knowledge-retrieval","display_name":"factual knowledge retrieval","score":0.6330000162124634},{"id":"https://openalex.org/keywords/information-flow","display_name":"information flow","score":0.7379999756813049},{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.5910000205039978},{"id":"https://openalex.org/keywords/attribute-extraction","display_name":"attribute extraction","score":0.6779999732971191},{"id":"https://openalex.org/keywords/knowledge-localization","display_name":"knowledge localization","score":0.7990000247955322},{"id":"https://openalex.org/keywords/knowledge-editing","display_name":"knowledge editing","score":0.9580000042915344}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7680454850196838},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.6940394639968872},{"id":"https://openalex.org/C2777855551","wikidata":"https://www.wikidata.org/wiki/Q12310021","display_name":"Subject (documents)","level":2,"score":0.6059361100196838},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.5980925559997559},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.579331636428833},{"id":"https://openalex.org/C100660578","wikidata":"https://www.wikidata.org/wiki/Q18733","display_name":"Recall","level":2,"score":0.556921124458313},{"id":"https://openalex.org/C25343380","wikidata":"https://www.wikidata.org/wiki/Q277521","display_name":"Relation (database)","level":2,"score":0.5357761383056641},{"id":"https://openalex.org/C153604712","wikidata":"https://www.wikidata.org/wiki/Q7310755","display_name":"Relationship extraction","level":3,"score":0.519980251789093},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5148088335990906},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4791911840438843},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.46783486008644104},{"id":"https://openalex.org/C195807954","wikidata":"https://www.wikidata.org/wiki/Q1662562","display_name":"Information extraction","level":2,"score":0.3917519748210907},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.2573226988315582},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.12676337361335754},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09731215238571167},{"id":"https://openalex.org/C161191863","wikidata":"https://www.wikidata.org/wiki/Q199655","display_name":"Library science","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2023.emnlp-main.751","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.751","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.751","pdf_url":"https://aclanthology.org/2023.emnlp-main.751.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2304.14767","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2304.14767","pdf_url":"https://arxiv.org/pdf/2304.14767","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2304.14767","doi":"https://doi.org/10.48550/arxiv.2304.14767","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2304.14767","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.emnlp-main.751","doi":"https://doi.org/10.18653/v1/2023.emnlp-main.751","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2023.emnlp-main.751","pdf_url":"https://aclanthology.org/2023.emnlp-main.751.pdf","source":{"id":"https://openalex.org/S4363608991","display_name":"Conference on Empirical Methods in Natural Language Processing (EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4389520380.pdf","grobid_xml":"https://content.openalex.org/works/W4389520380.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W574030736","https://openalex.org/W1902674502","https://openalex.org/W2034132675","https://openalex.org/W2058602429","https://openalex.org/W2603222250","https://openalex.org/W2626778328","https://openalex.org/W2946359678","https://openalex.org/W2946417913","https://openalex.org/W2964159778","https://openalex.org/W2970476646","https://openalex.org/W2970727289","https://openalex.org/W2970820321","https://openalex.org/W2972854536","https://openalex.org/W2991382858","https://openalex.org/W3008686018","https://openalex.org/W3037932933","https://openalex.org/W3092292656","https://openalex.org/W3097252660","https://openalex.org/W3117576675","https://openalex.org/W3152884768","https://openalex.org/W3154575616","https://openalex.org/W3206867351","https://openalex.org/W4294955582","https://openalex.org/W4306313145","https://openalex.org/W4308023630","https://openalex.org/W4316135772","https://openalex.org/W4362598605","https://openalex.org/W4385570354","https://openalex.org/W4385571791","https://openalex.org/W4385572504","https://openalex.org/W4385572928","https://openalex.org/W4385572952","https://openalex.org/W4385573190","https://openalex.org/W4386566424","https://openalex.org/W4386566794","https://openalex.org/W4386566901","https://openalex.org/W7133236469","https://openalex.org/W7133239630"],"related_works":["https://openalex.org/W4416035582","https://openalex.org/W4409347883","https://openalex.org/W4391800998","https://openalex.org/W4281657280","https://openalex.org/W7166862972","https://openalex.org/W4393336146","https://openalex.org/W4407686153","https://openalex.org/W4385573668","https://openalex.org/W4307411724","https://openalex.org/W4415797265"],"abstract_inverted_index":{"Transformer-based":[0],"language":[1],"models":[2],"(LMs)":[3],"are":[4,20,29,175],"known":[5,25],"to":[6,58,76,124,134,144],"capture":[7],"factual":[8,18,173],"knowledge":[9,186],"in":[10,162,180],"their":[11,163],"parameters.While":[12],"previous":[13],"work":[14],"looked":[15],"into":[16],"where":[17,73],"associations":[19,174],"stored,":[21],"only":[22],"little":[23],"is":[24,151],"about":[26,53],"how":[27,48,172],"they":[28],"retrieved":[30],"internally":[31,179],"during":[32],"inference.We":[33],"investigate":[34],"this":[35,149],"question":[36],"through":[37,114],"the":[38,49,54,60,77,81,88,93,107,110,120,131,135,137,141,146],"lens":[39],"of":[40,171],"information":[41,52,74,94,129],"flow.Given":[42],"a":[43,100,168],"subject-relation":[44],"query,":[45],"we":[46,67,98],"study":[47],"model":[50],"aggregates":[51],"subject":[55,89,143],"and":[56,177,188],"relation":[57,82,132],"predict":[59],"correct":[61],"attribute.With":[62],"interventions":[63],"on":[64,185],"attention":[65,155],"edges,":[66],"first":[68],"identify":[69],"two":[70],"critical":[71],"points":[72],"propagates":[75,133],"prediction:":[78],"one":[79],"from":[80,87,130],"positions":[83],"followed":[84],"by":[85,91,119],"another":[86],"positions.Next,":[90],"analyzing":[92],"at":[95,109],"these":[96],"points,":[97],"unveil":[99],"three-step":[101],"internal":[102],"mechanism":[103],"for":[104],"attribute":[105],"extraction.First,":[106],"representation":[108,139],"lastsubject":[111],"position":[112],"goes":[113],"an":[115],"enrichment":[116],"process,":[117],"driven":[118],"early":[121],"MLP":[122],"sublayers,":[123],"encode":[125,159],"many":[126],"subject-related":[127],"attributes.Second,":[128],"prediction.Third,":[136],"prediction":[138],"\"queries\"":[140],"enriched":[142],"extract":[145],"attribute.Perhaps":[147],"surprisingly,":[148],"extraction":[150],"typically":[152],"done":[153],"via":[154],"heads,":[156],"which":[157],"often":[158],"subject-attribute":[160],"mappings":[161],"parameters.Overall,":[164],"our":[165],"findings":[166],"introduce":[167],"comprehensive":[169],"view":[170],"stored":[176],"extracted":[178],"LMs,":[181],"facilitating":[182],"future":[183],"research":[184],"localization":[187],"editing.":[189],"1":[190]},"counts_by_year":[{"year":2026,"cited_by_count":101},{"year":2025,"cited_by_count":101},{"year":2024,"cited_by_count":74},{"year":2023,"cited_by_count":31}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-12-11T00:00:00"}