{"id":"https://openalex.org/W4385573607","doi":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","title":"Transformer Language Models without Positional Encodings Still Learn Positional Information","display_name":"Transformer Language Models without Positional Encodings Still Learn Positional Information","publication_year":2022,"publication_date":"2022-01-01","ids":{"openalex":"https://openalex.org/W4385573607","doi":"https://doi.org/10.18653/v1/2022.findings-emnlp.99"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2022.findings-emnlp.99","doi":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","pdf_url":"https://aclanthology.org/2022.findings-emnlp.99.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2022","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2022.findings-emnlp.99.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065238853","display_name":"Adi Haviv","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Adi Haviv","raw_affiliation_strings":["Tel Aviv University","Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I16391192","https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058640064","display_name":"Ori Ram","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Ori Ram","raw_affiliation_strings":["Tel Aviv University","Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I16391192","https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035538068","display_name":"Ofir Press","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Ofir Press","raw_affiliation_strings":["Tel Aviv University","\u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"\u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I201448701"]},{"raw_affiliation_string":"Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I16391192","https://openalex.org/I201448701"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081126295","display_name":"Peter Izsak","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Peter Izsak","raw_affiliation_strings":["Tel Aviv University","Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I16391192","https://openalex.org/I201448701"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024311574","display_name":"Omer Levy","orcid":"https://orcid.org/0000-0001-7300-8191","observed_orcids":["https://orcid.org/0000-0001-7300-8191"]},"institutions":[{"id":"https://openalex.org/I16391192","display_name":"Tel Aviv University","ror":"https://ror.org/04mhzgx49","country_code":"IL","type":"education","lineage":["https://openalex.org/I16391192"]},{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["IL","US"],"is_corresponding":false,"raw_author_name":"Omer Levy","raw_affiliation_strings":["Tel Aviv University","Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tel Aviv University","institution_ids":["https://openalex.org/I16391192"]},{"raw_affiliation_string":"Tel Aviv University \u03c9 University of Washington \u03b9 Intel Labs \u00b5 Meta AI","institution_ids":["https://openalex.org/I16391192","https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.5977,"has_fulltext":true,"cited_by_count":61,"citation_normalized_percentile":{"value":0.98205521,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1382","last_page":"1390"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9437999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.04470000043511391,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.0024999999441206455,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/positional-encoding","display_name":"positional encoding","score":0.7110000252723694},{"id":"https://openalex.org/keywords/causal-transformer","display_name":"causal transformer","score":0.5189999938011169},{"id":"https://openalex.org/keywords/causal-attention","display_name":"causal attention","score":0.6460000276565552},{"id":"https://openalex.org/keywords/probing","display_name":"probing","score":0.3799999952316284},{"id":"https://openalex.org/keywords/sequence-length","display_name":"sequence length","score":0.5370000004768372},{"id":"https://openalex.org/keywords/transformer-based-language-models","display_name":"transformer language models","score":0.6690000295639038}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7457433342933655},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6783631443977356},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6148717403411865},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5593274235725403},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.538352370262146},{"id":"https://openalex.org/C44291984","wikidata":"https://www.wikidata.org/wiki/Q1074173","display_name":"Question answering","level":2,"score":0.5098311305046082},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.43451988697052},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.432628870010376},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2022.findings-emnlp.99","doi":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","pdf_url":"https://aclanthology.org/2022.findings-emnlp.99.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2022","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2203.16634","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2203.16634","pdf_url":"https://arxiv.org/pdf/2203.16634","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2203.16634","doi":"https://doi.org/10.48550/arxiv.2203.16634","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2203.16634","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2022.findings-emnlp.99","doi":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2022.findings-emnlp.99","pdf_url":"https://aclanthology.org/2022.findings-emnlp.99.pdf","source":{"id":"https://openalex.org/S7407096085","display_name":"Findings of the Association for Computational Linguistics: EMNLP (Findings of EMNLP)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EMNLP 2022","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320307102","display_name":"Intel Corporation","ror":"https://ror.org/01ek73717"},{"id":"https://openalex.org/F4320319290","display_name":"Meta","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4385573607.pdf","grobid_xml":"https://content.openalex.org/works/W4385573607.grobid-xml"},"referenced_works_count":20,"referenced_works":["https://openalex.org/W1522301498","https://openalex.org/W2133564696","https://openalex.org/W2525332836","https://openalex.org/W2613904329","https://openalex.org/W2626778328","https://openalex.org/W2789541106","https://openalex.org/W2894175714","https://openalex.org/W2933138175","https://openalex.org/W2943845043","https://openalex.org/W2951008357","https://openalex.org/W2963341956","https://openalex.org/W2965373594","https://openalex.org/W2969945254","https://openalex.org/W2970820321","https://openalex.org/W2981852735","https://openalex.org/W2988945824","https://openalex.org/W3030163527","https://openalex.org/W3118781290","https://openalex.org/W3152698349","https://openalex.org/W3196318247"],"related_works":["https://openalex.org/W4406052031","https://openalex.org/W7202177609","https://openalex.org/W4224911293","https://openalex.org/W4412889704","https://openalex.org/W4385573098","https://openalex.org/W4385570072","https://openalex.org/W7134837129","https://openalex.org/W7123174759","https://openalex.org/W3173814445","https://openalex.org/W4406032125"],"abstract_inverted_index":{"Causal":[0],"transformer":[1],"language":[2],"models":[3,52],"(LMs),":[4],"such":[5,15,51],"as":[6,16],"GPT-3,":[7],"typically":[8],"require":[9],"some":[10],"form":[11],"of":[12,57,80,114],"positional":[13,17,26,100],"encoding,":[14],"embeddings.However,":[18],"we":[19],"show":[20],"that":[21,35,50,70,82,95],"LMs":[22,97],"without":[23],"any":[24],"explicit":[25,106],"encoding":[27],"are":[28],"still":[29],"competitive":[30],"with":[31],"standard":[32],"models,":[33],"and":[34,45],"this":[36],"phenomenon":[37],"is":[38],"robust":[39],"across":[40],"different":[41],"datasets,":[42],"model":[43,75],"sizes,":[44],"sequence":[46],"lengths.Probing":[47],"experiments":[48],"reveal":[49],"acquire":[53],"an":[54],"implicit":[55],"notion":[56],"absolute":[58,91],"positions":[59],"throughout":[60],"the":[61,66,74,78,105,112,115],"network,":[62],"effectively":[63],"compensating":[64],"for":[65],"missing":[67],"information.We":[68],"conjecture":[69],"causal":[71,96,116],"attention":[72],"enables":[73],"to":[76],"infer":[77],"number":[79],"predecessors":[81],"each":[83],"token":[84],"can":[85],"attend":[86],"to,":[87],"thereby":[88],"approximating":[89],"its":[90],"position.Our":[92],"findings":[93],"indicate":[94],"might":[98],"derive":[99],"awareness":[101],"not":[102],"only":[103],"from":[104,111],"positioning":[107],"mechanism,":[108],"but":[109],"also":[110],"effects":[113],"mask.":[117]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":17},{"year":2023,"cited_by_count":18},{"year":2022,"cited_by_count":2},{"year":2020,"cited_by_count":1}],"updated_date":"2026-10-29T09:56:32.521200","created_date":"2023-08-05T00:00:00"}