{"id":"https://openalex.org/W4381803920","doi":"https://doi.org/10.1162/tacl_a_00565","title":"Transparency Helps Reveal When Language Models Learn Meaning","display_name":"Transparency Helps Reveal When Language Models Learn Meaning","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4381803920","doi":"https://doi.org/10.1162/tacl_a_00565"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00565","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00565","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00565/2138350/tacl_a_00565.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00565/2138350/tacl_a_00565.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5036524439","display_name":"Zhaofeng Wu","orcid":"https://orcid.org/0000-0002-1470-5648"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhaofeng Wu","raw_affiliation_strings":["MIT. zfw@csail.mit.edu","MIT New York University Allen Institute for Artificial Intelligence","School of Computer Science & Engineering, University of Washington"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MIT. zfw@csail.mit.edu","institution_ids":[]},{"raw_affiliation_string":"MIT New York University Allen Institute for Artificial Intelligence","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"School of Computer Science & Engineering, University of Washington","institution_ids":["https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109946254","display_name":"William Merrill","orcid":null},"institutions":[{"id":"https://openalex.org/I57206974","display_name":"New York University","ror":"https://ror.org/0190ak572","country_code":"US","type":"education","lineage":["https://openalex.org/I57206974"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"William Merrill","raw_affiliation_strings":["New York University. willm@nyu.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"New York University. willm@nyu.edu","institution_ids":["https://openalex.org/I57206974"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100740618","display_name":"Hao Peng","orcid":"https://orcid.org/0000-0001-7422-630X"},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Hao Peng","raw_affiliation_strings":["Allen Institute for Artificial Intelligence. haop@allenai.org"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence. haop@allenai.org","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090038537","display_name":"Iz Beltagy","orcid":null},"institutions":[{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Iz Beltagy","raw_affiliation_strings":["Allen Institute for Artificial Intelligence. beltagy@allenai.org"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence. beltagy@allenai.org","institution_ids":["https://openalex.org/I4210156221"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088517824","display_name":"Noah A. Smith","orcid":"https://orcid.org/0000-0002-2310-6380"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]},{"id":"https://openalex.org/I4210156221","display_name":"Allen Institute for Artificial Intelligence","ror":"https://ror.org/05w520734","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210156221"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Noah A. Smith","raw_affiliation_strings":["Allen Institute for Artificial Intelligence. noah@allenai.org","Paul G. Allen School of Computer Science & Engineering, University of Washington. noah@allenai.org"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for Artificial Intelligence. noah@allenai.org","institution_ids":["https://openalex.org/I4210156221"]},{"raw_affiliation_string":"Paul G. Allen School of Computer Science & Engineering, University of Washington. noah@allenai.org","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5088517824","https://openalex.org/A5090038537","https://openalex.org/A5100740618","https://openalex.org/A5109946254"],"corresponding_institution_ids":["https://openalex.org/I201448701","https://openalex.org/I4210156221","https://openalex.org/I57206974"],"apc_list":null,"apc_paid":null,"fwci":1.4598,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.85365419,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"11","issue":null,"first_page":"617","last_page":"634"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.851588249206543},{"id":"https://openalex.org/keywords/transparency","display_name":"Transparency (behavior)","score":0.807832658290863},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.6837525963783264},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.6755213141441345},{"id":"https://openalex.org/keywords/meaning","display_name":"Meaning (existential)","score":0.6754646897315979},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5839134454727173},{"id":"https://openalex.org/keywords/context","display_name":"Context (archaeology)","score":0.5746593475341797},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5036606192588806},{"id":"https://openalex.org/keywords/semantics","display_name":"Semantics (computer science)","score":0.5032257437705994},{"id":"https://openalex.org/keywords/natural-language-understanding","display_name":"Natural language understanding","score":0.4688469171524048},{"id":"https://openalex.org/keywords/natural","display_name":"Natural (archaeology)","score":0.4125932455062866},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.3776973485946655},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.12473410367965698},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.09229275584220886}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.851588249206543},{"id":"https://openalex.org/C2780233690","wikidata":"https://www.wikidata.org/wiki/Q535347","display_name":"Transparency (behavior)","level":2,"score":0.807832658290863},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.6837525963783264},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.6755213141441345},{"id":"https://openalex.org/C2780876879","wikidata":"https://www.wikidata.org/wiki/Q3054749","display_name":"Meaning (existential)","level":2,"score":0.6754646897315979},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5839134454727173},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.5746593475341797},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5036606192588806},{"id":"https://openalex.org/C184337299","wikidata":"https://www.wikidata.org/wiki/Q1437428","display_name":"Semantics (computer science)","level":2,"score":0.5032257437705994},{"id":"https://openalex.org/C2779439875","wikidata":"https://www.wikidata.org/wiki/Q1078276","display_name":"Natural language understanding","level":3,"score":0.4688469171524048},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4125932455062866},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3776973485946655},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.12473410367965698},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.09229275584220886},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00565","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00565","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00565/2138350/tacl_a_00565.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:476b5a06e2e84082827eac4be7eaf3e8","is_oa":true,"landing_page_url":"https://doaj.org/article/476b5a06e2e84082827eac4be7eaf3e8","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 11, Pp 617-634 (2023)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00565","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00565","pdf_url":"https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl_a_00565/2138350/tacl_a_00565.pdf","source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7699999809265137}],"awards":[{"id":"https://openalex.org/G5693414920","display_name":null,"funder_award_id":"1922658","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4381803920.pdf","grobid_xml":"https://content.openalex.org/works/W4381803920.grobid-xml"},"referenced_works_count":65,"referenced_works":["https://openalex.org/W565760909","https://openalex.org/W1622302876","https://openalex.org/W1840435438","https://openalex.org/W1981818098","https://openalex.org/W2069778239","https://openalex.org/W2071089353","https://openalex.org/W2118373646","https://openalex.org/W2131354971","https://openalex.org/W2164548001","https://openalex.org/W2264742718","https://openalex.org/W2312747799","https://openalex.org/W2326326187","https://openalex.org/W2549835527","https://openalex.org/W2560852577","https://openalex.org/W2946359678","https://openalex.org/W2963341956","https://openalex.org/W2963383094","https://openalex.org/W2963751529","https://openalex.org/W2964204621","https://openalex.org/W2964303116","https://openalex.org/W2970120757","https://openalex.org/W2970476646","https://openalex.org/W2970641574","https://openalex.org/W2970820321","https://openalex.org/W2970862333","https://openalex.org/W2988217457","https://openalex.org/W3004346089","https://openalex.org/W3034723486","https://openalex.org/W3044438666","https://openalex.org/W3098267758","https://openalex.org/W3099178230","https://openalex.org/W3099700870","https://openalex.org/W3100283070","https://openalex.org/W3100475986","https://openalex.org/W3105816068","https://openalex.org/W3106889297","https://openalex.org/W3111186274","https://openalex.org/W3138301265","https://openalex.org/W3157301371","https://openalex.org/W3166986030","https://openalex.org/W3169229441","https://openalex.org/W3169602049","https://openalex.org/W3173673636","https://openalex.org/W3173798466","https://openalex.org/W3174693310","https://openalex.org/W3183859557","https://openalex.org/W3188660305","https://openalex.org/W3202546170","https://openalex.org/W3203259592","https://openalex.org/W4211208542","https://openalex.org/W4211245804","https://openalex.org/W4225571961","https://openalex.org/W4248708129","https://openalex.org/W4385573607","https://openalex.org/W4391156274","https://openalex.org/W6634895090","https://openalex.org/W6683955330","https://openalex.org/W6723682835","https://openalex.org/W6728271538","https://openalex.org/W6748344979","https://openalex.org/W6757635932","https://openalex.org/W6757817989","https://openalex.org/W6759833453","https://openalex.org/W6778883912","https://openalex.org/W6787040549"],"related_works":["https://openalex.org/W2367925007","https://openalex.org/W3015724364","https://openalex.org/W4288263119","https://openalex.org/W2967994095","https://openalex.org/W2900126711","https://openalex.org/W4285240985","https://openalex.org/W4225162083","https://openalex.org/W3202115945","https://openalex.org/W2542958340","https://openalex.org/W4286930972"],"abstract_inverted_index":{"Abstract":[0],"Many":[1],"current":[2,99],"NLP":[3],"systems":[4],"are":[5,68],"built":[6],"from":[7],"language":[8,55,75,100,106,120],"models":[9,56,101],"trained":[10],"to":[11,59,70,82,92,114],"optimize":[12],"unsupervised":[13],"objectives":[14],"on":[15],"large":[16],"amounts":[17],"of":[18,96,118],"raw":[19],"text.":[20],"Under":[21],"what":[22],"conditions":[23],"might":[24],"such":[25],"a":[26,88],"procedure":[27],"acquire":[28],"meaning?":[29],"Our":[30],"systematic":[31],"experiments":[32,86],"with":[33,38,48,73,87],"synthetic":[34],"data":[35],"reveal":[36],"that,":[37],"languages":[39,47],"where":[40],"all":[41],"expressions":[42],"have":[43],"context-independent":[44],"denotations":[45,67],"(i.e.,":[46],"strong":[49],"transparency),":[50],"both":[51],"autoregressive":[52],"and":[53],"masked":[54],"successfully":[57],"learn":[58],"emulate":[60],"semantic":[61],"relations":[62],"between":[63],"expressions.":[64],"However,":[65],"when":[66],"changed":[69],"be":[71],"context-dependent":[72,116],"the":[74,93,115],"otherwise":[76],"unmodified,":[77],"this":[78,111],"ability":[79],"degrades.":[80],"Turning":[81],"natural":[83,105,119],"language,":[84],"our":[85],"specific":[89],"phenomenon\u2014referential":[90],"opacity\u2014add":[91],"growing":[94],"body":[95],"evidence":[97],"that":[98],"do":[102],"not":[103],"represent":[104],"semantics":[107],"well.":[108],"We":[109],"show":[110],"failure":[112],"relates":[113],"nature":[117],"form-meaning":[121],"mappings.":[122]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1}],"updated_date":"2026-06-18T10:00:31.954636","created_date":"2025-10-10T00:00:00"}