{"id":"https://openalex.org/W3091922147","doi":"https://doi.org/10.18653/v1/2020.conll-1.45","title":"Are Pretrained Language Models Symbolic Reasoners over Knowledge?","display_name":"Are Pretrained Language Models Symbolic Reasoners over Knowledge?","publication_year":2020,"publication_date":"2020-01-01","ids":{"openalex":"https://openalex.org/W3091922147","doi":"https://doi.org/10.18653/v1/2020.conll-1.45","mag":"3091922147"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2020.conll-1.45","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.conll-1.45","pdf_url":"https://www.aclweb.org/anthology/2020.conll-1.45.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th Conference on Computational Natural Language Learning","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.aclweb.org/anthology/2020.conll-1.45.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019973471","display_name":"Nora Kassner","orcid":null},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":true,"raw_author_name":"Nora Kassner","raw_affiliation_strings":["Center for Information and Language Processing (CIS) LMU Munich, Germany","Ludwig-Maximilian University of Munich"],"affiliations":[{"raw_affiliation_string":"Center for Information and Language Processing (CIS) LMU Munich, Germany","institution_ids":["https://openalex.org/I8204097"]},{"raw_affiliation_string":"Ludwig-Maximilian University of Munich","institution_ids":["https://openalex.org/I8204097"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073456064","display_name":"Benno Krojer","orcid":null},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Benno Krojer","raw_affiliation_strings":["Center for Information and Language Processing (CIS) LMU Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Center for Information and Language Processing (CIS) LMU Munich, Germany","institution_ids":["https://openalex.org/I8204097"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5071144367","display_name":"Hinrich Sch\u00fctze","orcid":null},"institutions":[{"id":"https://openalex.org/I8204097","display_name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","ror":"https://ror.org/05591te55","country_code":"DE","type":"education","lineage":["https://openalex.org/I8204097"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Hinrich Sch\u00fctze","raw_affiliation_strings":["Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen, Munich, Germany"],"affiliations":[{"raw_affiliation_string":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen, Munich, Germany","institution_ids":["https://openalex.org/I8204097"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5019973471"],"corresponding_institution_ids":["https://openalex.org/I8204097"],"apc_list":null,"apc_paid":null,"fwci":0.4078,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.70313938,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"552","last_page":"564"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9909999966621399,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memorization","display_name":"Memorization","score":0.6996868252754211},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6583713889122009},{"id":"https://openalex.org/keywords/schema","display_name":"Schema (genetic algorithms)","score":0.6547929644584656},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5251227021217346},{"id":"https://openalex.org/keywords/analogical-reasoning","display_name":"Analogical reasoning","score":0.5107169151306152},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.4732661247253418},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.4209345579147339},{"id":"https://openalex.org/keywords/cognitive-science","display_name":"Cognitive science","score":0.40076035261154175},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.38823992013931274},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.32640349864959717},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32521724700927734},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.26679301261901855},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.14803335070610046},{"id":"https://openalex.org/keywords/analogy","display_name":"Analogy","score":0.13557428121566772},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08138620853424072}],"concepts":[{"id":"https://openalex.org/C30038468","wikidata":"https://www.wikidata.org/wiki/Q4354775","display_name":"Memorization","level":2,"score":0.6996868252754211},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6583713889122009},{"id":"https://openalex.org/C52146309","wikidata":"https://www.wikidata.org/wiki/Q7431116","display_name":"Schema (genetic algorithms)","level":2,"score":0.6547929644584656},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5251227021217346},{"id":"https://openalex.org/C2985612853","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogical reasoning","level":3,"score":0.5107169151306152},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4732661247253418},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.4209345579147339},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.40076035261154175},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.38823992013931274},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.32640349864959717},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32521724700927734},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.26679301261901855},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.14803335070610046},{"id":"https://openalex.org/C521332185","wikidata":"https://www.wikidata.org/wiki/Q185816","display_name":"Analogy","level":2,"score":0.13557428121566772},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08138620853424072},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.18653/v1/2020.conll-1.45","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.conll-1.45","pdf_url":"https://www.aclweb.org/anthology/2020.conll-1.45.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th Conference on Computational Natural Language Learning","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2006.10413","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.10413","pdf_url":"https://arxiv.org/pdf/2006.10413","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:3091922147","is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2006.10413.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2006.10413","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2006.10413","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.18653/v1/2020.conll-1.45","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2020.conll-1.45","pdf_url":"https://www.aclweb.org/anthology/2020.conll-1.45.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 24th Conference on Computational Natural Language Learning","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.6000000238418579,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G3467264548","display_name":null,"funder_award_id":"01IS18036A","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G3914568705","display_name":null,"funder_award_id":"This work was funded by the German Federal Ministr","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"},{"id":"https://openalex.org/G5106512922","display_name":null,"funder_award_id":"Deutsche Forschungsgemeinschaft (DFG","funder_id":"https://openalex.org/F4320320879","funder_display_name":"Deutsche Forschungsgemeinschaft"},{"id":"https://openalex.org/G7225624288","display_name":null,"funder_award_id":"This work was","funder_id":"https://openalex.org/F4320321114","funder_display_name":"Bundesministerium f\u00fcr Bildung und Forschung"}],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"},{"id":"https://openalex.org/F4320321114","display_name":"Bundesministerium f\u00fcr Bildung und Forschung","ror":"https://ror.org/04pz7b180"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3091922147.pdf","grobid_xml":"https://content.openalex.org/works/W3091922147.grobid-xml"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1970575036","https://openalex.org/W2121001250","https://openalex.org/W2268733724","https://openalex.org/W2889787757","https://openalex.org/W2909137510","https://openalex.org/W2919420119","https://openalex.org/W2951286828","https://openalex.org/W2962727366","https://openalex.org/W2962736243","https://openalex.org/W2962790689","https://openalex.org/W2963341956","https://openalex.org/W2963403868","https://openalex.org/W2965373594","https://openalex.org/W2969809624","https://openalex.org/W2970476646","https://openalex.org/W2971107062","https://openalex.org/W2997789497","https://openalex.org/W2997897037","https://openalex.org/W2998696444","https://openalex.org/W3004346089","https://openalex.org/W3007672467","https://openalex.org/W3017374003","https://openalex.org/W3034830866","https://openalex.org/W3034995113","https://openalex.org/W3035097102","https://openalex.org/W3035718362","https://openalex.org/W3111372685","https://openalex.org/W3137695714"],"related_works":["https://openalex.org/W3101204082","https://openalex.org/W3036486510","https://openalex.org/W3210661875","https://openalex.org/W2995993311","https://openalex.org/W2952828155","https://openalex.org/W2972869776","https://openalex.org/W2909534157","https://openalex.org/W3034144525","https://openalex.org/W21520908","https://openalex.org/W1493420784","https://openalex.org/W3214335899","https://openalex.org/W3040224409","https://openalex.org/W3176825161","https://openalex.org/W2807085696","https://openalex.org/W2620626032","https://openalex.org/W2592312659","https://openalex.org/W3178680873","https://openalex.org/W1993979041","https://openalex.org/W2966070748","https://openalex.org/W2989150936"],"abstract_inverted_index":{"How":[0],"can":[1],"pretrained":[2],"language":[3],"models":[4],"(PLMs)":[5],"learn":[6,68],"factual":[7],"knowledge":[8],"from":[9],"the":[10,15,29,41,46,58,88],"training":[11,53],"set?":[12],"We":[13],"investigate":[14],"two":[16],"most":[17],"important":[18],"mechanisms:":[19],"reasoning":[20,73,92],"and":[21,54,108],"memorization.":[22],"Prior":[23],"work":[24],"has":[25],"attempted":[26],"to":[27,67,69],"quantify":[28],"number":[30],"of":[31,90],"facts":[32,50,55],"PLMs":[33,65],"learn,":[34],"but":[35,76],"we":[36,62,98],"present,":[37],"using":[38],"synthetic":[39],"data,":[40],"first":[42],"study":[43],"that":[44,64,86],"investigates":[45],"causal":[47],"relation":[48],"between":[49],"present":[51],"in":[52],"learned":[56,91],"by":[57,105],"PLM.":[59],"For":[60,96],"reasoning,":[61],"show":[63],"seem":[66],"apply":[70],"some":[71],"symbolic":[72],"rules":[74,93],"correctly":[75],"struggle":[77],"with":[78],"others,":[79],"including":[80],"two-hop":[81],"reasoning.":[82],"Further":[83],"analysis":[84],"suggests":[85],"even":[87],"application":[89],"is":[94],"flawed.":[95],"memorization,":[97],"identify":[99],"schema":[100],"conformity":[101],"(facts":[102],"systematically":[103],"supported":[104],"other":[106],"facts)":[107],"frequency":[109],"as":[110],"key":[111],"factors":[112],"for":[113],"its":[114],"success.":[115]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-04-10T15:06:20.359241","created_date":"2025-10-10T00:00:00"}