{"id":"https://openalex.org/W3031914912","doi":"https://doi.org/10.1073/pnas.1907367117","title":"Emergent linguistic structure in artificial neural networks trained by self-supervision","display_name":"Emergent linguistic structure in artificial neural networks trained by self-supervision","publication_year":2020,"publication_date":"2020-06-03","ids":{"openalex":"https://openalex.org/W3031914912","doi":"https://doi.org/10.1073/pnas.1907367117","mag":"3031914912","pmid":"https://pubmed.ncbi.nlm.nih.gov/32493748"},"language":"en","primary_location":{"id":"doi:10.1073/pnas.1907367117","is_oa":false,"landing_page_url":"https://doi.org/10.1073/pnas.1907367117","pdf_url":null,"source":{"id":"https://openalex.org/S125754415","display_name":"Proceedings of the National Academy of Sciences","issn_l":"0027-8424","issn":["0027-8424","1091-6490"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320052","host_organization_name":"National Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310320052"],"host_organization_lineage_names":["National Academy of Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7720155","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046006076","display_name":"Christopher D. Manning","orcid":"https://orcid.org/0000-0001-6155-649X"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Christopher D. Manning","raw_affiliation_strings":["Computer Science Department, Stanford University, Stanford, CA 94305;"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, Stanford, CA 94305;","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049064405","display_name":"Kevin B. Clark","orcid":"https://orcid.org/0000-0003-0544-2150"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kevin Clark","raw_affiliation_strings":["Computer Science Department, Stanford University, Stanford, CA 94305;"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, Stanford, CA 94305;","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075075495","display_name":"John Hewitt","orcid":"https://orcid.org/0000-0003-1320-6633"},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John Hewitt","raw_affiliation_strings":["Computer Science Department, Stanford University, Stanford, CA 94305;"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, Stanford, CA 94305;","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5088072227","display_name":"Urvashi Khandelwal","orcid":null},"institutions":[{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Urvashi Khandelwal","raw_affiliation_strings":["Computer Science Department, Stanford University, Stanford, CA 94305;"],"affiliations":[{"raw_affiliation_string":"Computer Science Department, Stanford University, Stanford, CA 94305;","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024311574","display_name":"Omer Levy","orcid":"https://orcid.org/0000-0001-7300-8191"},"institutions":[{"id":"https://openalex.org/I4210114444","display_name":"Meta (United States)","ror":"https://ror.org/01zbnvs85","country_code":"US","type":"company","lineage":["https://openalex.org/I4210114444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Omer Levy","raw_affiliation_strings":["Facebook Artificial Intelligence Research, Facebook Inc., Seattle, WA 98109"],"affiliations":[{"raw_affiliation_string":"Facebook Artificial Intelligence Research, Facebook Inc., Seattle, WA 98109","institution_ids":["https://openalex.org/I4210114444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5046006076"],"corresponding_institution_ids":["https://openalex.org/I97018004"],"apc_list":null,"apc_paid":null,"fwci":28.1368,"has_fulltext":false,"cited_by_count":332,"citation_normalized_percentile":{"value":0.99721033,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"117","issue":"48","first_page":"30046","last_page":"30054"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9975000023841858,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7524679899215698},{"id":"https://openalex.org/keywords/coreference","display_name":"Coreference","score":0.710125207901001},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6630575060844421},{"id":"https://openalex.org/keywords/parsing","display_name":"Parsing","score":0.5958060622215271},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.5891603827476501},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5754978656768799},{"id":"https://openalex.org/keywords/tree-structure","display_name":"Tree structure","score":0.5333703756332397},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5208821296691895},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.4825131595134735},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.46526575088500977},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.37426167726516724},{"id":"https://openalex.org/keywords/resolution","display_name":"Resolution (logic)","score":0.30989474058151245},{"id":"https://openalex.org/keywords/data-structure","display_name":"Data structure","score":0.1608838438987732}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7524679899215698},{"id":"https://openalex.org/C28076734","wikidata":"https://www.wikidata.org/wiki/Q63087","display_name":"Coreference","level":3,"score":0.710125207901001},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6630575060844421},{"id":"https://openalex.org/C186644900","wikidata":"https://www.wikidata.org/wiki/Q194152","display_name":"Parsing","level":2,"score":0.5958060622215271},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.5891603827476501},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5754978656768799},{"id":"https://openalex.org/C163797641","wikidata":"https://www.wikidata.org/wiki/Q2067937","display_name":"Tree structure","level":3,"score":0.5333703756332397},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5208821296691895},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.4825131595134735},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.46526575088500977},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.37426167726516724},{"id":"https://openalex.org/C138268822","wikidata":"https://www.wikidata.org/wiki/Q1051925","display_name":"Resolution (logic)","level":2,"score":0.30989474058151245},{"id":"https://openalex.org/C162319229","wikidata":"https://www.wikidata.org/wiki/Q175263","display_name":"Data structure","level":2,"score":0.1608838438987732},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1073/pnas.1907367117","is_oa":false,"landing_page_url":"https://doi.org/10.1073/pnas.1907367117","pdf_url":null,"source":{"id":"https://openalex.org/S125754415","display_name":"Proceedings of the National Academy of Sciences","issn_l":"0027-8424","issn":["0027-8424","1091-6490"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320052","host_organization_name":"National Academy of Sciences","host_organization_lineage":["https://openalex.org/P4310320052"],"host_organization_lineage_names":["National Academy of Sciences"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences","raw_type":"journal-article"},{"id":"pmid:32493748","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/32493748","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the National Academy of Sciences of the United States of America","raw_type":null},{"id":"pmh:oai:pubmedcentral.nih.gov:7720155","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7720155","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc Natl Acad Sci U S A","raw_type":"Text"}],"best_oa_location":{"id":"pmh:oai:pubmedcentral.nih.gov:7720155","is_oa":true,"landing_page_url":"https://www.ncbi.nlm.nih.gov/pmc/articles/7720155","pdf_url":null,"source":{"id":"https://openalex.org/S2764455111","display_name":"PubMed Central","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proc Natl Acad Sci U S A","raw_type":"Text"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8399999737739563,"display_name":"Quality Education"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320309327","display_name":"Google","ror":"https://ror.org/00njsd438"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":78,"referenced_works":["https://openalex.org/W49348824","https://openalex.org/W1508977358","https://openalex.org/W1514535095","https://openalex.org/W1517853909","https://openalex.org/W1632114991","https://openalex.org/W1902237438","https://openalex.org/W1924770834","https://openalex.org/W1983578042","https://openalex.org/W2020944885","https://openalex.org/W2050423830","https://openalex.org/W2072128103","https://openalex.org/W2073454546","https://openalex.org/W2084920657","https://openalex.org/W2092654472","https://openalex.org/W2119439969","https://openalex.org/W2129657639","https://openalex.org/W2129945114","https://openalex.org/W2133564696","https://openalex.org/W2140324965","https://openalex.org/W2153579005","https://openalex.org/W2155069789","https://openalex.org/W2172806452","https://openalex.org/W2180093461","https://openalex.org/W2250539671","https://openalex.org/W2250861254","https://openalex.org/W2264869955","https://openalex.org/W2270070752","https://openalex.org/W2306852879","https://openalex.org/W2549835527","https://openalex.org/W2552110825","https://openalex.org/W2559655401","https://openalex.org/W2563574619","https://openalex.org/W2579343286","https://openalex.org/W2596567068","https://openalex.org/W2604272474","https://openalex.org/W2605717780","https://openalex.org/W2607303097","https://openalex.org/W2612690371","https://openalex.org/W2750779823","https://openalex.org/W2768763386","https://openalex.org/W2781528640","https://openalex.org/W2788510939","https://openalex.org/W2795355797","https://openalex.org/W2798727047","https://openalex.org/W2799124508","https://openalex.org/W2888329843","https://openalex.org/W2888922637","https://openalex.org/W2893141505","https://openalex.org/W2896457183","https://openalex.org/W2908854766","https://openalex.org/W2910243263","https://openalex.org/W2914924671","https://openalex.org/W2922565841","https://openalex.org/W2962788148","https://openalex.org/W2962811598","https://openalex.org/W2963341956","https://openalex.org/W2963571341","https://openalex.org/W2963651521","https://openalex.org/W2963661253","https://openalex.org/W2963751529","https://openalex.org/W2964204621","https://openalex.org/W2964303116","https://openalex.org/W2970727289","https://openalex.org/W2972324944","https://openalex.org/W2990138404","https://openalex.org/W3102226577","https://openalex.org/W4231109964","https://openalex.org/W4238252461","https://openalex.org/W4245765565","https://openalex.org/W4254300870","https://openalex.org/W6623517193","https://openalex.org/W6691431627","https://openalex.org/W6739901393","https://openalex.org/W6748634344","https://openalex.org/W6750696111","https://openalex.org/W6754744501","https://openalex.org/W6767594909","https://openalex.org/W6948152991"],"related_works":["https://openalex.org/W2139373276","https://openalex.org/W1509033667","https://openalex.org/W2227889443","https://openalex.org/W4385749782","https://openalex.org/W3167631113","https://openalex.org/W2145164276","https://openalex.org/W2004630825","https://openalex.org/W2765988220","https://openalex.org/W1568205884","https://openalex.org/W1586415134"],"abstract_inverted_index":{"This":[0],"paper":[1],"explores":[2],"the":[3,18,156],"knowledge":[4],"of":[5,37,61,76,96,138,155],"linguistic":[6,108],"structure":[7,110],"learned":[8,139],"by":[9,71,162],"large":[10,174],"artificial":[11,113],"neural":[12,114],"networks,":[13],"trained":[14],"via":[15,35],"self-supervision,":[16],"whereby":[17],"model":[19],"simply":[20],"tries":[21],"to":[22,148],"predict":[23],"a":[24,28,58,135,149],"masked":[25],"word":[26],"in":[27,112,120,141],"given":[29],"context.":[30],"Human":[31],"language":[32,40,63,91],"communication":[33],"is":[34],"sequences":[36],"words,":[38],"but":[39],"understanding":[41],"requires":[42],"constructing":[43],"rich":[44],"hierarchical":[45,109],"structures":[46,159],"that":[47,87,118,134],"are":[48],"never":[49],"observed":[50],"explicitly.":[51],"The":[52],"mechanisms":[53],"for":[54,80,106],"this":[55,81,97],"have":[56,171],"been":[57],"prime":[59],"mystery":[60],"human":[62],"acquisition,":[64],"while":[65],"engineering":[66],"work":[67],"has":[68],"mainly":[69],"proceeded":[70],"supervised":[72],"learning":[73],"on":[74,124],"treebanks":[75],"sentences":[77],"hand":[78],"labeled":[79],"latent":[82],"structure.":[83],"However,":[84],"we":[85,132],"demonstrate":[86,117],"modern":[88],"deep":[89],"contextual":[90],"models":[92,122,143,170],"learn":[93],"major":[94],"aspects":[95],"structure,":[98],"without":[99],"any":[100],"explicit":[101],"supervision.":[102],"We":[103],"develop":[104],"methods":[105],"identifying":[107],"emergent":[111],"networks":[115],"and":[116,128],"components":[119],"these":[121,142,169],"focus":[123],"syntactic":[125],"grammatical":[126],"relationships":[127],"anaphoric":[129],"coreference.":[130],"Indeed,":[131],"show":[133],"linear":[136],"transformation":[137],"embeddings":[140],"captures":[144],"parse":[145],"tree":[146,158],"distances":[147],"surprising":[150],"degree,":[151],"allowing":[152],"approximate":[153],"reconstruction":[154],"sentence":[157],"normally":[160],"assumed":[161],"linguists.":[163],"These":[164],"results":[165],"help":[166],"explain":[167],"why":[168],"brought":[172],"such":[173],"improvements":[175],"across":[176],"many":[177],"language-understanding":[178],"tasks.":[179]},"counts_by_year":[{"year":2026,"cited_by_count":14},{"year":2025,"cited_by_count":50},{"year":2024,"cited_by_count":58},{"year":2023,"cited_by_count":85},{"year":2022,"cited_by_count":59},{"year":2021,"cited_by_count":49},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":3}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}