{"id":"https://openalex.org/W4376112422","doi":"https://doi.org/10.1017/s1351324923000128","title":"What should be encoded by position embedding for neural network language models?","display_name":"What should be encoded by position embedding for neural network language models?","publication_year":2023,"publication_date":"2023-05-10","ids":{"openalex":"https://openalex.org/W4376112422","doi":"https://doi.org/10.1017/s1351324923000128"},"language":"en","primary_location":{"id":"doi:10.1017/s1351324923000128","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1017/s1351324923000128","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/2875F94FCDAD65A6DAD34664027D1642/S1351324923000128a.pdf/div-class-title-what-should-be-encoded-by-position-embedding-for-neural-network-language-models-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/2875F94FCDAD65A6DAD34664027D1642/S1351324923000128a.pdf/div-class-title-what-should-be-encoded-by-position-embedding-for-neural-network-language-models-div.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5020871289","display_name":"Shuiyuan Yu","orcid":"https://orcid.org/0000-0003-2474-6686"},"institutions":[{"id":"https://openalex.org/I115212828","display_name":"Beijing Language and Culture University","ror":"https://ror.org/03te2zs36","country_code":"CN","type":"education","lineage":["https://openalex.org/I115212828"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shuiyuan Yu","raw_affiliation_strings":["Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","institution_ids":["https://openalex.org/I115212828"]},{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,","institution_ids":["https://openalex.org/I115212828"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100404241","display_name":"Zihao Zhang","orcid":"https://orcid.org/0000-0002-9290-8125"},"institutions":[{"id":"https://openalex.org/I115212828","display_name":"Beijing Language and Culture University","ror":"https://ror.org/03te2zs36","country_code":"CN","type":"education","lineage":["https://openalex.org/I115212828"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zihao Zhang","raw_affiliation_strings":["Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","institution_ids":["https://openalex.org/I115212828"]},{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,","institution_ids":["https://openalex.org/I115212828"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100413180","display_name":"Haitao Liu","orcid":"https://orcid.org/0000-0003-1724-4418"},"institutions":[{"id":"https://openalex.org/I115212828","display_name":"Beijing Language and Culture University","ror":"https://ror.org/03te2zs36","country_code":"CN","type":"education","lineage":["https://openalex.org/I115212828"]},{"id":"https://openalex.org/I1284762954","display_name":"Zhejiang A & F University","ror":"https://ror.org/02vj4rn06","country_code":"CN","type":"education","lineage":["https://openalex.org/I1284762954"]},{"id":"https://openalex.org/I186272606","display_name":"Guangdong University of Foreign Studies","ror":"https://ror.org/00fhc9y79","country_code":"CN","type":"education","lineage":["https://openalex.org/I186272606"]},{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Haitao Liu","raw_affiliation_strings":["Centre for Linguistics and Applied Linguistics, Guangdong University of Foreign Studies, Guangzhou 510006, P. R. China","Department of Linguistics, Zhejiang University, Hangzhou 310058, P. R. China","Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,","Department of Linguistics, Zhejiang University, Hangzhou 310058, P. R. China, and"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centre for Linguistics and Applied Linguistics, Guangdong University of Foreign Studies, Guangzhou 510006, P. R. China","institution_ids":["https://openalex.org/I186272606"]},{"raw_affiliation_string":"Department of Linguistics, Zhejiang University, Hangzhou 310058, P. R. China","institution_ids":["https://openalex.org/I1284762954"]},{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China","institution_ids":["https://openalex.org/I115212828"]},{"raw_affiliation_string":"Institute of Quantitative Linguistics, Beijing Language and Culture University, Beijing 100083, P. R. China,","institution_ids":["https://openalex.org/I115212828"]},{"raw_affiliation_string":"Department of Linguistics, Zhejiang University, Hangzhou 310058, P. R. China, and","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5100413180"],"corresponding_institution_ids":["https://openalex.org/I115212828","https://openalex.org/I1284762954","https://openalex.org/I186272606","https://openalex.org/I76130692"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":{"value":0.04311684,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"30","issue":"2","first_page":"294","last_page":"318"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7229495048522949},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.7063698768615723},{"id":"https://openalex.org/keywords/word-order","display_name":"Word order","score":0.7026191353797913},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.634853720664978},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.5728511214256287},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5595175623893738},{"id":"https://openalex.org/keywords/encode","display_name":"ENCODE","score":0.5572729110717773},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5450087785720825},{"id":"https://openalex.org/keywords/word-embedding","display_name":"Word embedding","score":0.5135069489479065},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5065163373947144},{"id":"https://openalex.org/keywords/word","display_name":"Word (group theory)","score":0.5016787052154541},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.43097835779190063},{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.42476779222488403},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.39664795994758606},{"id":"https://openalex.org/keywords/linguistics","display_name":"Linguistics","score":0.21929848194122314},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.06969895958900452}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7229495048522949},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.7063698768615723},{"id":"https://openalex.org/C70777604","wikidata":"https://www.wikidata.org/wiki/Q257885","display_name":"Word order","level":2,"score":0.7026191353797913},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.634853720664978},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5728511214256287},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5595175623893738},{"id":"https://openalex.org/C66746571","wikidata":"https://www.wikidata.org/wiki/Q1134833","display_name":"ENCODE","level":3,"score":0.5572729110717773},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5450087785720825},{"id":"https://openalex.org/C2777462759","wikidata":"https://www.wikidata.org/wiki/Q18395344","display_name":"Word embedding","level":3,"score":0.5135069489479065},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5065163373947144},{"id":"https://openalex.org/C90805587","wikidata":"https://www.wikidata.org/wiki/Q10944557","display_name":"Word (group theory)","level":2,"score":0.5016787052154541},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.43097835779190063},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.42476779222488403},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.39664795994758606},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.21929848194122314},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.06969895958900452},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1017/s1351324923000128","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1017/s1351324923000128","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/2875F94FCDAD65A6DAD34664027D1642/S1351324923000128a.pdf/div-class-title-what-should-be-encoded-by-position-embedding-for-neural-network-language-models-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1017/s1351324923000128","is_oa":true,"landing_page_url":"http://dx.doi.org/10.1017/s1351324923000128","pdf_url":"https://www.cambridge.org/core/services/aop-cambridge-core/content/view/2875F94FCDAD65A6DAD34664027D1642/S1351324923000128a.pdf/div-class-title-what-should-be-encoded-by-position-embedding-for-neural-network-language-models-div.pdf","source":{"id":"https://openalex.org/S18088403","display_name":"Natural Language Engineering","issn_l":"1351-3249","issn":["1351-3249","1469-8110"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310311721","host_organization_name":"Cambridge University Press","host_organization_lineage":["https://openalex.org/P4310311721","https://openalex.org/P4310311702"],"host_organization_lineage_names":["Cambridge University Press","University of Cambridge"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Natural Language Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.8399999737739563,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G5372615172","display_name":null,"funder_award_id":"20YJ140010","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G682248771","display_name":null,"funder_award_id":"20AYY021","funder_id":"https://openalex.org/F4320335869","funder_display_name":"National Social Science Fund of China"}],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"},{"id":"https://openalex.org/F4320325046","display_name":"Beijing Language and Culture University","ror":"https://ror.org/03te2zs36"},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null},{"id":"https://openalex.org/F4320335869","display_name":"National Social Science Fund of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4376112422.pdf"},"referenced_works_count":41,"referenced_works":["https://openalex.org/W1508165687","https://openalex.org/W1970092758","https://openalex.org/W1986533001","https://openalex.org/W2007858530","https://openalex.org/W2019588402","https://openalex.org/W2019858696","https://openalex.org/W2030007850","https://openalex.org/W2041258965","https://openalex.org/W2087148916","https://openalex.org/W2092845274","https://openalex.org/W2100506586","https://openalex.org/W2136646664","https://openalex.org/W2142509733","https://openalex.org/W2147152072","https://openalex.org/W2158600149","https://openalex.org/W2250539671","https://openalex.org/W2334424438","https://openalex.org/W2882319491","https://openalex.org/W2922523190","https://openalex.org/W2941599692","https://openalex.org/W2956530858","https://openalex.org/W2962739339","https://openalex.org/W2963925437","https://openalex.org/W2970550739","https://openalex.org/W2971087717","https://openalex.org/W2973154008","https://openalex.org/W2999905431","https://openalex.org/W3031914912","https://openalex.org/W3101791345","https://openalex.org/W3102094970","https://openalex.org/W3105248300","https://openalex.org/W3106210592","https://openalex.org/W3107606501","https://openalex.org/W3174169056","https://openalex.org/W3210914950","https://openalex.org/W4233835482","https://openalex.org/W4297945192","https://openalex.org/W6673578895","https://openalex.org/W6737778391","https://openalex.org/W6753056052","https://openalex.org/W6788864986"],"related_works":["https://openalex.org/W2468279273","https://openalex.org/W2354198838","https://openalex.org/W1989130879","https://openalex.org/W2103419012","https://openalex.org/W2988126442","https://openalex.org/W1974414866","https://openalex.org/W2063982682","https://openalex.org/W2338543196","https://openalex.org/W2911655849","https://openalex.org/W4286432911"],"abstract_inverted_index":{"Abstract":[0],"Word":[1],"order":[2,79,164],"is":[3,38,64,171],"one":[4,19,121],"of":[5,20,45,52,54,68,84,103,122,126,137,149,189],"the":[6,12,21,61,82,88,97,101,123,135,146,197],"most":[7,22],"important":[8],"grammatical":[9,157],"devices":[10],"and":[11,105,129,139,179],"basis":[13],"for":[14],"language":[15],"understanding.":[16],"However,":[17],"as":[18],"popular":[23],"NLP":[24,62],"architectures,":[25],"Transformer":[26],"does":[27],"not":[28,172],"explicitly":[29],"encode":[30],"word":[31,106,140,150],"order.":[32],"A":[33],"solution":[34],"to":[35,39,80,165,174,186,196],"this":[36],"problem":[37],"incorporate":[40],"position":[41,46,56,73,85,154],"information":[42,57,74,86,191],"by":[43],"means":[44],"encoding/embedding":[47],"(PE).":[48],"Although":[49],"a":[50,127],"variety":[51],"methods":[53],"incorporating":[55],"have":[58,184],"been":[59],"proposed,":[60],"community":[63],"still":[65],"in":[66,75,92,108,163],"want":[67],"detailed":[69],"statistical":[70],"researches":[71],"on":[72,87,177],"real-life":[76],"language.":[77],"In":[78],"understand":[81],"influence":[83],"correlation":[89],"between":[90],"words":[91,104,138],"more":[93,187],"detail,":[94],"we":[95,143],"investigated":[96],"factors":[98],"that":[99,114,145,162],"affect":[100,134],"frequency":[102,136,147],"sequences":[107,151],"large":[109],"corpora.":[110],"Our":[111,159],"results":[112],"show":[113],"absolute":[115,178],"position,":[116,118],"relative":[117,153,180],"being":[119],"at":[120],"two":[124],"ends":[125],"sentence":[128,130],"length":[131],"all":[132],"significantly":[133],"sequences.":[141],"Besides,":[142],"observed":[144],"distribution":[148],"over":[152],"carries":[155],"valuable":[156],"information.":[158],"study":[160],"suggests":[161],"accurately":[166],"capture":[167],"word\u2013word":[168],"correlations,":[169],"it":[170],"enough":[173],"focus":[175],"merely":[176],"position.":[181],"Transformers":[182],"should":[183],"access":[185],"types":[188],"position-related":[190],"which":[192],"may":[193],"require":[194],"improvements":[195],"current":[198],"architecture.":[199]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}