{"id":"https://openalex.org/W3028871071","doi":"https://doi.org/10.48550/arxiv.2006.01095","title":"Emergence of Separable Manifolds in Deep Language Representations","display_name":"Emergence of Separable Manifolds in Deep Language Representations","publication_year":2020,"publication_date":"2020-06-01","ids":{"openalex":"https://openalex.org/W3028871071","doi":"https://doi.org/10.48550/arxiv.2006.01095","mag":"3028871071"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2006.01095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.01095","pdf_url":"https://arxiv.org/pdf/2006.01095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2006.01095","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Mamou, Jonathan","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1343180700","display_name":"Intel (United States)","ror":"https://ror.org/01ek73717","country_code":"US","type":"company","lineage":["https://openalex.org/I1343180700"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mamou, Jonathan","raw_affiliation_strings":["Intel Labs","Intel AI Labs"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intel Labs","institution_ids":["https://openalex.org/I1343180700"]},{"raw_affiliation_string":"Intel AI Labs","institution_ids":["https://openalex.org/I1343180700"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Le, Hang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Le, Hang","raw_affiliation_strings":["Massachusetts Institute of Technology","-MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"-MIT","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081125447","display_name":"Miguel Del Rio","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Del Rio, Miguel","raw_affiliation_strings":["-MIT"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"-MIT","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080560015","display_name":"Cory Stephenson","orcid":"https://orcid.org/0000-0003-0078-6501","observed_orcids":["https://orcid.org/0000-0003-0078-6501"]},"institutions":[{"id":"https://openalex.org/I1343180700","display_name":"Intel (United States)","ror":"https://ror.org/01ek73717","country_code":"US","type":"company","lineage":["https://openalex.org/I1343180700"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephenson, Cory","raw_affiliation_strings":["Intel Labs","INTEL Corporation#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intel Labs","institution_ids":["https://openalex.org/I1343180700"]},{"raw_affiliation_string":"INTEL Corporation#TAB#","institution_ids":["https://openalex.org/I1343180700"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089829380","display_name":"Hanlin Tang","orcid":"https://orcid.org/0000-0002-6613-9174","observed_orcids":["https://orcid.org/0000-0002-6613-9174","https://orcid.org/0000-0003-0306-1637"]},"institutions":[{"id":"https://openalex.org/I1343180700","display_name":"Intel (United States)","ror":"https://ror.org/01ek73717","country_code":"US","type":"company","lineage":["https://openalex.org/I1343180700"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tang, Hanlin","raw_affiliation_strings":["Intel Labs","Intel AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intel Labs","institution_ids":["https://openalex.org/I1343180700"]},{"raw_affiliation_string":"Intel AI","institution_ids":["https://openalex.org/I1343180700"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100693798","display_name":"Yoon Kim","orcid":"https://orcid.org/0000-0002-4837-8411","observed_orcids":["https://orcid.org/0000-0002-4837-8411"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kim, Yoon","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5016533438","display_name":"SueYeon Chung","orcid":"https://orcid.org/0000-0001-5726-4865","observed_orcids":["https://orcid.org/0000-0001-5726-4865"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]},{"id":"https://openalex.org/I78577930","display_name":"Columbia University","ror":"https://ror.org/00hj8s172","country_code":"US","type":"education","lineage":["https://openalex.org/I78577930"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chung, SueYeon","raw_affiliation_strings":["Columbia University ","Massachusetts Institute of Technology","Massachusetts Institute Of Technology#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Columbia University ","institution_ids":["https://openalex.org/I78577930"]},{"raw_affiliation_string":"Massachusetts Institute of Technology","institution_ids":["https://openalex.org/I63966007"]},{"raw_affiliation_string":"Massachusetts Institute Of Technology#TAB#","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9164999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9164999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.010700000450015068,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.008100000210106373,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/linear-separability","display_name":"linear separability","score":0.828000009059906},{"id":"https://openalex.org/keywords/part-of-speech-tags","display_name":"part-of-speech tags","score":0.8029999732971191},{"id":"https://openalex.org/keywords/ambiguous-words","display_name":"ambiguous words","score":0.49399998784065247},{"id":"https://openalex.org/keywords/deep-language-models","display_name":"deep language models","score":0.4830000102519989},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.7319999933242798},{"id":"https://openalex.org/keywords/roberta","display_name":"RoBERTa","score":0.9710000157356262},{"id":"https://openalex.org/keywords/gpt","display_name":"GPT","score":0.9789999723434448},{"id":"https://openalex.org/keywords/model-depth","display_name":"model depth","score":0.3199999928474426}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.577132523059845},{"id":"https://openalex.org/C529865628","wikidata":"https://www.wikidata.org/wiki/Q1790740","display_name":"Manifold (fluid mechanics)","level":2,"score":0.5138097405433655},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.4569741487503052},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3949625492095947},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.3432946503162384},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.16425666213035583},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2006.01095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.01095","pdf_url":"https://arxiv.org/pdf/2006.01095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2006.01095","doi":"https://doi.org/10.48550/arxiv.2006.01095","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2006.01095","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2006.01095","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2006.01095","pdf_url":"https://arxiv.org/pdf/2006.01095","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4253858762","display_name":"NeuroNex Theory Team: Columbia University Theoretical Neuroscience Center","funder_award_id":"1707398","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3028871071.pdf","grobid_xml":"https://content.openalex.org/works/W3028871071.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W1579838312","https://openalex.org/W1632114991","https://openalex.org/W2030450972","https://openalex.org/W2040036684","https://openalex.org/W2058616551","https://openalex.org/W2153633111","https://openalex.org/W2161278885","https://openalex.org/W2176287621","https://openalex.org/W2177153902","https://openalex.org/W2525907473","https://openalex.org/W2626778328","https://openalex.org/W2766572821","https://openalex.org/W2767204723","https://openalex.org/W2787560479","https://openalex.org/W2794002979","https://openalex.org/W2801092741","https://openalex.org/W2899047292","https://openalex.org/W2914924671","https://openalex.org/W2915130814","https://openalex.org/W2922565841","https://openalex.org/W2925209818","https://openalex.org/W2941161300","https://openalex.org/W2946359678","https://openalex.org/W2946857231","https://openalex.org/W2947024452","https://openalex.org/W2948947170","https://openalex.org/W2951891110","https://openalex.org/W2953369973","https://openalex.org/W2963341956","https://openalex.org/W2965373594","https://openalex.org/W2970421120","https://openalex.org/W2970727289","https://openalex.org/W2970820321","https://openalex.org/W2970862333","https://openalex.org/W2970866842","https://openalex.org/W2975059944","https://openalex.org/W2978017171","https://openalex.org/W3004639598"],"related_works":["https://openalex.org/W7214520607","https://openalex.org/W7131066352","https://openalex.org/W3039556919","https://openalex.org/W3004639598","https://openalex.org/W4285340778","https://openalex.org/W4384108749","https://openalex.org/W4408189760","https://openalex.org/W2948337921","https://openalex.org/W4414580992","https://openalex.org/W7128538935"],"abstract_inverted_index":{"Deep":[0],"neural":[1,39,80],"networks":[2],"(DNNs)":[3],"have":[4,45,64],"shown":[5],"much":[6],"empirical":[7],"success":[8],"in":[9,41,61,79,145,170],"solving":[10],"perceptual":[11],"tasks":[12],"across":[13,136],"various":[14],"cognitive":[15,58],"modalities.":[16],"While":[17],"they":[18,63],"are":[19],"only":[20],"loosely":[21],"inspired":[22],"by":[23,175],"the":[24,42,165],"biological":[25],"brain,":[26],"recent":[27,92],"studies":[28],"report":[29],"considerable":[30],"similarities":[31],"between":[32],"representations":[33,102,111,119],"extracted":[34],"from":[35,94,112,120],"task-optimized":[36],"DNNs":[37,44],"and":[38,60,128,183],"populations":[40],"brain.":[43],"subsequently":[46],"become":[47],"a":[48,68,91,176],"popular":[49],"model":[50,122],"class":[51],"to":[52,76,108],"infer":[53],"computational":[54,95],"principles":[55],"underlying":[56],"complex":[57],"functions,":[59],"turn,":[62],"also":[65],"emerged":[66],"as":[67],"natural":[69],"testbed":[70],"for":[71,131,141],"applying":[72],"methods":[73],"originally":[74],"developed":[75],"probe":[77],"information":[78],"populations.":[81],"In":[82,160],"this":[83],"work,":[84],"we":[85,162],"utilize":[86],"mean-field":[87],"theoretic":[88],"manifold":[89],"analysis,":[90],"technique":[93],"neuroscience":[96],"that":[97,164],"connects":[98],"geometry":[99],"of":[100,106,133,167,179],"feature":[101],"with":[103,150],"linear":[104,168],"separability":[105,169],"classes,":[107],"analyze":[109],"language":[110],"large-scale":[113],"contextual":[114],"embedding":[115],"models.":[116],"We":[117],"explore":[118],"different":[121],"families":[123],"(BERT,":[124],"RoBERTa,":[125],"GPT,":[126],"etc.)":[127],"find":[129,163],"evidence":[130],"emergence":[132,166],"linguistic":[134],"manifolds":[135,140,172],"layer":[137],"depth":[138],"(e.g.,":[139],"part-of-speech":[142,152,155],"tags),":[143],"especially":[144],"ambiguous":[146],"data":[147],"(i.e,":[148],"words":[149],"multiple":[151],"tags,":[153],"or":[154],"classes":[156],"including":[157],"many":[158],"words).":[159],"addition,":[161],"these":[171],"is":[173],"driven":[174],"combined":[177],"reduction":[178],"manifolds'":[180],"radius,":[181],"dimensionality":[182],"inter-manifold":[184],"correlations.":[185]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":7},{"year":2022,"cited_by_count":5},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":1}],"updated_date":"2026-11-05T12:25:35.769452","created_date":"2020-06-05T00:00:00"}