{"id":"https://openalex.org/W4313447747","doi":"https://doi.org/10.48550/arxiv.2212.14852","title":"An Analysis of Attention via the Lens of Exchangeability and Latent Variable Models","display_name":"An Analysis of Attention via the Lens of Exchangeability and Latent Variable Models","publication_year":2022,"publication_date":"2022-12-30","ids":{"openalex":"https://openalex.org/W4313447747","doi":"https://doi.org/10.48550/arxiv.2212.14852"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2212.14852","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.14852","pdf_url":"https://arxiv.org/pdf/2212.14852","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2212.14852","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100459698","display_name":"Yufeng Zhang","orcid":"https://orcid.org/0000-0003-3293-4759","observed_orcids":["https://orcid.org/0000-0003-3293-4759"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhang, Yufeng","raw_affiliation_strings":["Northwestern University ;","\u2016"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University ;","institution_ids":["https://openalex.org/I111979921"]},{"raw_affiliation_string":"\u2016","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Liu, Boyi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Boyi","raw_affiliation_strings":["Northwestern University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University ;","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Cai, Qi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Cai, Qi","raw_affiliation_strings":["Northwestern University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University ;","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100779971","display_name":"Lingxiao Wang","orcid":"https://orcid.org/0000-0002-1654-4681","observed_orcids":["https://orcid.org/0000-0002-1654-4681"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Lingxiao","raw_affiliation_strings":["Northwestern University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University ;","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101934111","display_name":"Zhaoran Wang","orcid":"https://orcid.org/0000-0002-6617-4842","observed_orcids":["https://orcid.org/0000-0002-6617-4842","https://orcid.org/0000-0002-2789-7909"]},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Zhaoran","raw_affiliation_strings":["Northwestern University ;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Northwestern University ;","institution_ids":["https://openalex.org/I111979921"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I111979921"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":23,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9092000126838684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9092000126838684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.014999999664723873,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.008500000461935997,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/attention-mechanism","display_name":"attention mechanism","score":0.9800000190734863},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.9760000109672546},{"id":"https://openalex.org/keywords/exchangeability","display_name":"exchangeability","score":0.9620000123977661},{"id":"https://openalex.org/keywords/latent-variable-modeling","display_name":"latent variable models","score":0.9959999918937683},{"id":"https://openalex.org/keywords/relational-inference","display_name":"relational inference","score":0.6740000247955322},{"id":"https://openalex.org/keywords/positional-encoding","display_name":"positional encoding","score":0.7680000066757202},{"id":"https://openalex.org/keywords/posterior-distribution","display_name":"posterior distribution","score":0.45100000500679016},{"id":"https://openalex.org/keywords/empirical-risk-minimization","display_name":"empirical risk minimization","score":0.8100000023841858},{"id":"https://openalex.org/keywords/self-supervised-learning","display_name":"self-supervised learning","score":0.9819999933242798},{"id":"https://openalex.org/keywords/condition-number","display_name":"condition number","score":0.7319999933242798},{"id":"https://openalex.org/keywords/generalization-error","display_name":"generalization error","score":0.7070000171661377},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.47600001096725464},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.6480000019073486},{"id":"https://openalex.org/keywords/vision-transformer","display_name":"vision transformer","score":0.6380000114440918}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7564350366592407},{"id":"https://openalex.org/C51167844","wikidata":"https://www.wikidata.org/wiki/Q4422623","display_name":"Latent variable","level":2,"score":0.714402973651886},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6576846241950989},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5529756546020508},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.483568400144577},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.48130762577056885},{"id":"https://openalex.org/C182365436","wikidata":"https://www.wikidata.org/wiki/Q50701","display_name":"Variable (mathematics)","level":2,"score":0.43040311336517334},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42721492052078247},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.33231380581855774},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3223288655281067},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.23706260323524475},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2212.14852","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.14852","pdf_url":"https://arxiv.org/pdf/2212.14852","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2212.14852","doi":"https://doi.org/10.48550/arxiv.2212.14852","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2212.14852","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2212.14852","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2212.14852","pdf_url":"https://arxiv.org/pdf/2212.14852","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4313447747.pdf","grobid_xml":"https://content.openalex.org/works/W4313447747.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2418306335","https://openalex.org/W2565538933","https://openalex.org/W2797262859","https://openalex.org/W2805516822","https://openalex.org/W2894604724","https://openalex.org/W2896457183","https://openalex.org/W2900959181","https://openalex.org/W2911109671","https://openalex.org/W2918794972","https://openalex.org/W2970100546","https://openalex.org/W2992043355","https://openalex.org/W3038074040","https://openalex.org/W3039502308","https://openalex.org/W3089607359","https://openalex.org/W3094502228","https://openalex.org/W3112176259","https://openalex.org/W3157286395","https://openalex.org/W3170901997","https://openalex.org/W3185853715","https://openalex.org/W3206006178","https://openalex.org/W3208828709","https://openalex.org/W4221149947","https://openalex.org/W4281638513","https://openalex.org/W4305038462","https://openalex.org/W7133224310","https://openalex.org/W7133240241"],"related_works":["https://openalex.org/W7160695516","https://openalex.org/W2968219088","https://openalex.org/W4404344146","https://openalex.org/W4391632064","https://openalex.org/W7223564422","https://openalex.org/W3090952993","https://openalex.org/W4383567511","https://openalex.org/W3135593154","https://openalex.org/W7127337831","https://openalex.org/W7131861627"],"abstract_inverted_index":{"With":[0],"the":[1,10,32,53,57,61,73,77,85,130,147,151,181,185,206,210,213,245,261],"attention":[2,33,54,179,204],"mechanism,":[3],"transformers":[4,14],"achieve":[5],"significant":[6],"empirical":[7,240],"successes.":[8],"Despite":[9],"intuitive":[11],"understanding":[12],"that":[13,111,178,233,269],"perform":[15],"relational":[16,220],"inference":[17,221],"over":[18,222],"long":[19,223],"sequences":[20],"to":[21,71,114,189,243,249,272],"produce":[22],"desirable":[23,48,58,74],"representations,":[24],"we":[25,128,176,201,231,264],"lack":[26],"a":[27,47,67,107,133,144,159,250,266],"rigorous":[28],"theory":[29],"on":[30,126,174,229],"how":[31,203,217],"mechanism":[34,55],"achieves":[35],"it.":[36],"In":[37,141,199],"particular,":[38,142],"several":[39],"intriguing":[40],"questions":[41],"remain":[42],"open:":[43],"(a)":[44,125],"What":[45],"makes":[46],"representation?":[49],"(b)":[50,173],"How":[51,65],"does":[52,66],"infer":[56,72],"representation":[59,75,137,145],"within":[60],"forward":[62],"pass?":[63,79],"(c)":[64,228],"pretraining":[68],"procedure":[69],"learn":[70,244],"through":[76],"backward":[78],"We":[80],"observe":[81],"that,":[82],"as":[83],"is":[84,112,194,254,270],"case":[86],"in":[87,162,196,260],"BERT":[88],"and":[89,135,166,236],"ViT,":[90],"input":[91,115,139,155,197,257],"tokens":[92],"are":[93],"often":[94],"exchangeable":[95],"since":[96],"they":[97],"already":[98],"include":[99],"positional":[100],"encodings.":[101],"The":[102],"notion":[103],"of":[104,132,138,150,209,256],"exchangeability":[105],"induces":[106],"latent":[108,152,186],"variable":[109,153],"model":[110],"invariant":[113],"sizes,":[116],"which":[117,157,193,215,253],"enables":[118],"our":[119],"theoretical":[120],"analysis.":[121],"-":[122,170,225],"To":[123,171,226],"answer":[124,172,227],"representation,":[127],"establish":[129],"existence":[131],"sufficient":[134],"minimal":[136],"tokens.":[140],"such":[143],"instantiates":[146],"posterior":[148,187],"distribution":[149],"given":[154,212],"tokens,":[156],"plays":[158],"central":[160],"role":[161],"predicting":[163],"output":[164],"labels":[165],"solving":[167,273],"downstream":[168,274],"tasks.":[169,275],"inference,":[175],"prove":[177,232],"with":[180],"desired":[182,246],"parameter":[183,247],"infers":[184],"up":[188,248],"an":[190],"approximation":[191],"error,":[192,252],"decreasing":[195],"sizes.":[198,258],"detail,":[200],"quantify":[202],"approximates":[205],"conditional":[207],"mean":[208],"value":[211],"key,":[214],"characterizes":[216],"it":[218],"performs":[219],"sequences.":[224],"learning,":[230],"both":[234],"supervised":[235],"self-supervised":[237,262],"objectives":[238],"allow":[239],"risk":[241],"minimization":[242],"generalization":[251],"independent":[255],"Particularly,":[259],"setting,":[263],"identify":[265],"condition":[267],"number":[268],"pivotal":[271]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":5}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2023-01-06T00:00:00"}