{"id":"https://openalex.org/W4391881499","doi":"https://doi.org/10.1007/s11063-024-11539-7","title":"Rethinking Position Embedding Methods in the Transformer Architecture","display_name":"Rethinking Position Embedding Methods in the Transformer Architecture","publication_year":2024,"publication_date":"2024-02-16","ids":{"openalex":"https://openalex.org/W4391881499","doi":"https://doi.org/10.1007/s11063-024-11539-7"},"language":"en","primary_location":{"id":"doi:10.1007/s11063-024-11539-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-024-11539-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-024-11539-7.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11063-024-11539-7.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100709813","display_name":"Xin Zhou","orcid":"https://orcid.org/0009-0002-6486-5639"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xin Zhou","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081838860","display_name":"Zhaohui Ren","orcid":"https://orcid.org/0000-0002-1390-1348"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaohui Ren","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103027236","display_name":"Shihua Zhou","orcid":"https://orcid.org/0000-0002-4829-1404"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shihua Zhou","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043136926","display_name":"Zeyu Jiang","orcid":"https://orcid.org/0000-0003-4529-2812"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeyu Jiang","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089556035","display_name":"Tianzhuang Yu","orcid":"https://orcid.org/0000-0002-9884-2446"},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"TianZhuang Yu","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013794772","display_name":"Hengfa Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I9224756","display_name":"Northeastern University","ror":"https://ror.org/03awzbc87","country_code":"CN","type":"education","lineage":["https://openalex.org/I9224756"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hengfa Luo","raw_affiliation_strings":["Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Schoool of Mechanical Engineering and Automation, Northeastern University, Wenhua Road, Shenyang, 110819, Liaoning, China","institution_ids":["https://openalex.org/I9224756"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5100709813"],"corresponding_institution_ids":["https://openalex.org/I9224756"],"apc_list":{"value":2390,"currency":"EUR","value_usd":2990},"apc_paid":{"value":2390,"currency":"EUR","value_usd":2990},"fwci":1.7777,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.84279716,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"56","issue":"2","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11343","display_name":"Power Transformer Diagnostics and Insulation","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11343","display_name":"Power Transformer Diagnostics and Insulation","score":0.9850999712944031,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11222","display_name":"Magnetic Properties and Applications","score":0.9689000248908997,"subfield":{"id":"https://openalex.org/subfields/2504","display_name":"Electronic, Optical and Magnetic Materials"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10511","display_name":"High voltage insulation and dielectric phenomena","score":0.9606999754905701,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/embedding","display_name":"Embedding","score":0.6110343337059021},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5582329034805298},{"id":"https://openalex.org/keywords/computational-intelligence","display_name":"Computational intelligence","score":0.4897036850452423},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.46842706203460693},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4619491696357727},{"id":"https://openalex.org/keywords/position","display_name":"Position (finance)","score":0.4557068943977356},{"id":"https://openalex.org/keywords/position-paper","display_name":"Position paper","score":0.42257702350616455},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2806146740913391},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.24914777278900146},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.236103355884552},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.19958451390266418},{"id":"https://openalex.org/keywords/art","display_name":"Art","score":0.12200909852981567},{"id":"https://openalex.org/keywords/world-wide-web","display_name":"World Wide Web","score":0.07771909236907959},{"id":"https://openalex.org/keywords/finance","display_name":"Finance","score":0.07211673259735107}],"concepts":[{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.6110343337059021},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5582329034805298},{"id":"https://openalex.org/C139502532","wikidata":"https://www.wikidata.org/wiki/Q1122090","display_name":"Computational intelligence","level":2,"score":0.4897036850452423},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.46842706203460693},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4619491696357727},{"id":"https://openalex.org/C198082294","wikidata":"https://www.wikidata.org/wiki/Q3399648","display_name":"Position (finance)","level":2,"score":0.4557068943977356},{"id":"https://openalex.org/C78780964","wikidata":"https://www.wikidata.org/wiki/Q7233193","display_name":"Position paper","level":2,"score":0.42257702350616455},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2806146740913391},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.24914777278900146},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.236103355884552},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.19958451390266418},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.12200909852981567},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.07771909236907959},{"id":"https://openalex.org/C10138342","wikidata":"https://www.wikidata.org/wiki/Q43015","display_name":"Finance","level":1,"score":0.07211673259735107},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s11063-024-11539-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-024-11539-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-024-11539-7.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s11063-024-11539-7","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11063-024-11539-7","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11063-024-11539-7.pdf","source":{"id":"https://openalex.org/S140962798","display_name":"Neural Processing Letters","issn_l":"1370-4621","issn":["1370-4621","1573-773X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Processing Letters","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Sustainable cities and communities","id":"https://metadata.un.org/sdg/11","score":0.6299999952316284}],"awards":[{"id":"https://openalex.org/G1784884431","display_name":null,"funder_award_id":"2022-MS-125","funder_id":"https://openalex.org/F4320323086","funder_display_name":"Natural Science Foundation of Liaoning Province"},{"id":"https://openalex.org/G5926930151","display_name":null,"funder_award_id":"52275091","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7372326643","display_name":null,"funder_award_id":"N2303011","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320323086","display_name":"Natural Science Foundation of Liaoning Province","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4391881499.pdf"},"referenced_works_count":30,"referenced_works":["https://openalex.org/W1977295328","https://openalex.org/W2108598243","https://openalex.org/W2138011018","https://openalex.org/W2194775991","https://openalex.org/W2533598788","https://openalex.org/W2565639579","https://openalex.org/W2618530766","https://openalex.org/W2962843773","https://openalex.org/W2963925437","https://openalex.org/W2964110616","https://openalex.org/W3035160371","https://openalex.org/W3096609285","https://openalex.org/W3097217077","https://openalex.org/W3131500599","https://openalex.org/W3137278571","https://openalex.org/W3138516171","https://openalex.org/W3170841864","https://openalex.org/W3203701986","https://openalex.org/W4206706211","https://openalex.org/W4214493665","https://openalex.org/W4214627427","https://openalex.org/W4214634256","https://openalex.org/W4250954493","https://openalex.org/W4255999337","https://openalex.org/W4312847199","https://openalex.org/W4313007769","https://openalex.org/W6600175266","https://openalex.org/W6600421821","https://openalex.org/W6601998182","https://openalex.org/W6607687417"],"related_works":["https://openalex.org/W2081900870","https://openalex.org/W2183306018","https://openalex.org/W2549990292","https://openalex.org/W2345479200","https://openalex.org/W2951819827","https://openalex.org/W2849310602","https://openalex.org/W2419146053","https://openalex.org/W2088247287","https://openalex.org/W3006008237","https://openalex.org/W2963903416"],"abstract_inverted_index":{"Abstract":[0],"In":[1],"the":[2,13,16,23,31,36,46,50,70,96,104,120,135,141,145,150],"transformer":[3],"architecture,":[4],"as":[5,129],"self-attention":[6,32],"reads":[7],"entire":[8],"image":[9],"patches":[10,19],"at":[11],"once,":[12],"context":[14],"of":[15,39,162],"sequence":[17],"between":[18,103],"is":[20,27,73],"omitted.":[21],"Therefore,":[22],"position":[24,47,80,101],"embedding":[25,106],"method":[26,72,122,152],"employed":[28],"to":[29,49,158],"assist":[30],"layers":[33],"in":[34,98,144],"computing":[35],"ordering":[37],"information":[38,102,127],"tokens.":[40],"While":[41],"many":[42],"papers":[43,59],"simply":[44],"add":[45],"vector":[48,53],"corresponding":[51],"token":[52,77],"rather":[54],"than":[55,134],"concatenating":[56],"them,":[57],"few":[58,147],"offer":[60],"a":[61,159],"thorough":[62],"explanation":[63],"and":[64,79,109,111,132,156],"comparison":[65],"beyond":[66],"dimension":[67],"reduction.":[68],"However,":[69],"addition":[71,136],"not":[74],"meaningful":[75],"because":[76],"vectors":[78,81],"are":[82],"different":[83],"physical":[84],"quantities":[85],"that":[86,119],"cannot":[87],"be":[88],"directly":[89],"combined":[90],"through":[91],"addition.":[92],"Hence,":[93],"we":[94],"investigate":[95],"disparity":[97],"learnable":[99],"absolute":[100],"two":[105],"methods":[107],"(concatenation":[108],"addition)":[110],"compare":[112],"their":[113],"performance":[114,160],"on":[115],"models.":[116],"Experiments":[117],"demonstrate":[118],"concatenation":[121,151],"can":[123],"learn":[124],"more":[125],"spatial":[126],"(such":[128],"horizontal,":[130],"vertical,":[131],"angle)":[133],"method.":[137],"Furthermore,":[138],"it":[139],"reduces":[140],"attention":[142],"distance":[143],"final":[146],"layers.":[148],"Moreover,":[149],"exhibits":[153],"greater":[154],"robustness":[155],"leads":[157],"gain":[161],"0.1\u20130.5%":[163],"for":[164],"existing":[165],"models":[166],"without":[167],"additional":[168],"computation":[169],"overhead.":[170]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2}],"updated_date":"2026-05-05T08:41:31.759640","created_date":"2025-10-10T00:00:00"}