{"id":"https://openalex.org/W4386566427","doi":"https://doi.org/10.18653/v1/2023.findings-eacl.176","title":"Better Pre-Training by Reducing Representation Confusion","display_name":"Better Pre-Training by Reducing Representation Confusion","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4386566427","doi":"https://doi.org/10.18653/v1/2023.findings-eacl.176"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2023.findings-eacl.176","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-eacl.176","pdf_url":"https://aclanthology.org/2023.findings-eacl.176.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2023","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://aclanthology.org/2023.findings-eacl.176.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100707830","display_name":"Haojie Zhang","orcid":"https://orcid.org/0000-0002-4577-3600"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]},{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haojie Zhang","raw_affiliation_strings":["Peking University, China","WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Peking University, China","institution_ids":["https://openalex.org/I20231570"]},{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027921204","display_name":"Mingfei Liang","orcid":"https://orcid.org/0000-0002-0599-1806"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingfei Liang","raw_affiliation_strings":["WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101577090","display_name":"Ruobing Xie","orcid":"https://orcid.org/0000-0003-3170-5647"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruobing Xie","raw_affiliation_strings":["WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010994296","display_name":"Zhenlong Sun","orcid":"https://orcid.org/0000-0001-5944-6064"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhenlong Sun","raw_affiliation_strings":["WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100335274","display_name":"Bo Zhang","orcid":"https://orcid.org/0000-0002-3555-8202"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bo Zhang","raw_affiliation_strings":["WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023086553","display_name":"Leyu Lin","orcid":"https://orcid.org/0000-0001-5471-500X"},"institutions":[{"id":"https://openalex.org/I2250653659","display_name":"Tencent (China)","ror":"https://ror.org/00hhjss72","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250653659"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Leyu Lin","raw_affiliation_strings":["WeChat Search Application Department, Tencent, China","{coldhjzhang, aesopliang,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"WeChat Search Application Department, Tencent, China","institution_ids":["https://openalex.org/I2250653659"]},{"raw_affiliation_string":"{coldhjzhang, aesopliang,","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2428,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.57037624,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"2325","last_page":"2336"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7823317050933838},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6617962718009949},{"id":"https://openalex.org/keywords/encoding","display_name":"Encoding (memory)","score":0.5864694118499756},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5772252082824707},{"id":"https://openalex.org/keywords/associative-property","display_name":"Associative property","score":0.551793098449707},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5345577001571655},{"id":"https://openalex.org/keywords/confusion","display_name":"Confusion","score":0.4841902554035187},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.43629199266433716},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.42747563123703003},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3996533155441284},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32887667417526245},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.11840051412582397}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7823317050933838},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6617962718009949},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.5864694118499756},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5772252082824707},{"id":"https://openalex.org/C159423971","wikidata":"https://www.wikidata.org/wiki/Q177251","display_name":"Associative property","level":2,"score":0.551793098449707},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5345577001571655},{"id":"https://openalex.org/C2781140086","wikidata":"https://www.wikidata.org/wiki/Q557945","display_name":"Confusion","level":2,"score":0.4841902554035187},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.43629199266433716},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.42747563123703003},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3996533155441284},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32887667417526245},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.11840051412582397},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C11171543","wikidata":"https://www.wikidata.org/wiki/Q41630","display_name":"Psychoanalysis","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2023.findings-eacl.176","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-eacl.176","pdf_url":"https://aclanthology.org/2023.findings-eacl.176.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2023","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2023.findings-eacl.176","is_oa":true,"landing_page_url":"http://dx.doi.org/10.18653/v1/2023.findings-eacl.176","pdf_url":"https://aclanthology.org/2023.findings-eacl.176.pdf","source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics: EACL 2023","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.7900000214576721,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386566427.pdf"},"referenced_works_count":29,"referenced_works":["https://openalex.org/W1566289585","https://openalex.org/W1614298861","https://openalex.org/W2119762594","https://openalex.org/W2250539671","https://openalex.org/W2397579082","https://openalex.org/W2587690726","https://openalex.org/W2759848268","https://openalex.org/W2768282280","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2946794439","https://openalex.org/W2963925437","https://openalex.org/W2963983698","https://openalex.org/W2964189376","https://openalex.org/W2965373594","https://openalex.org/W2970120757","https://openalex.org/W2970597249","https://openalex.org/W2970777192","https://openalex.org/W2971871542","https://openalex.org/W2972324944","https://openalex.org/W2996428491","https://openalex.org/W3011411500","https://openalex.org/W3122515622","https://openalex.org/W3122890974","https://openalex.org/W3175948724","https://openalex.org/W4287824654","https://openalex.org/W4288089799","https://openalex.org/W4385245566","https://openalex.org/W4388979610"],"related_works":["https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700","https://openalex.org/W2964976023","https://openalex.org/W1982477181","https://openalex.org/W2403083015"],"abstract_inverted_index":{"In":[0],"this":[1],"work,":[2],"we":[3,26,77,123],"revisit":[4],"the":[5,30,34,51,56,60,64,71,79,108,120,145,150,168,188,219],"Transformer-based":[6],"pre-trained":[7,130],"language":[8,131],"models":[9],"and":[10,21,40,63,98,116,139,149,178,212],"identify":[11],"two":[12,45,125,160],"different":[13,96,103,114,176,183,192],"types":[14],"of":[15,59,73,95,110,175,182,194,221],"information":[16,111,195,203],"confusion":[17,43,204],"in":[18,29,68,153,205],"position":[19,32,156],"encoding":[20,138],"model":[22,52,189],"representations,":[23],"respectively.":[24],"Firstly,":[25],"show":[27],"that":[28],"relative":[31,38,146,155],"encoding,":[33],"joint":[35],"modeling":[36],"about":[37],"distances":[39],"directions":[41],"brings":[42],"between":[44,170],"heterogeneous":[46],"information.":[47],"It":[48],"may":[49,106],"make":[50,107],"unable":[53],"to":[54,128,166,190,201],"capture":[55,191],"associative":[57],"semantics":[58],"same":[61],"distance":[62,147],"opposite":[65],"directions,":[66],"which":[67,105],"turn":[69],"affects":[70],"performance":[72],"downstream":[74],"tasks.":[75],"Secondly,":[76],"notice":[78],"BERT":[80],"with":[81],"Mask":[82],"Language":[83],"Modeling":[84],"(MLM)":[85],"pre-training":[86,141],"objective":[87],"outputs":[88],"similar":[89],"token":[90],"representations":[91,100],"(last":[92],"hidden":[93,173],"states":[94,174],"tokens)":[97],"head":[99],"(attention":[101],"weightsof":[102],"heads),":[104],"diversity":[109],"expressed":[112],"by":[113,119],"tokens":[115],"heads":[117],"limited.Motivated":[118],"above":[121],"investigation,":[122],"propose":[124],"novel":[126,161],"techniques":[127],"improve":[129],"models:":[132],"Decoupled":[133],"Directional":[134],"Relative":[135],"Position":[136],"(DDRP)":[137],"MTH":[140,158],"objective.":[142],"DDRP":[143],"decouples":[144],"features":[148,152],"directional":[151],"classical":[154],"encoding.":[157],"applies":[159],"auxiliary":[162],"regularizers":[163],"besides":[164],"MLM":[165],"enlarge":[167],"dissimilarities":[169],"(a)":[171],"last":[172],"tokens,":[177],"(b)":[179],"attention":[180],"weights":[181],"heads.":[184],"These":[185],"designs":[186],"allow":[187],"categories":[193],"more":[196],"clearly,":[197],"as":[198],"a":[199],"way":[200],"alleviate":[202],"representation":[206],"learning":[207],"for":[208],"better":[209],"optimization.Extensive":[210],"experiments":[211],"ablation":[213],"studies":[214],"on":[215],"GLUE":[216],"benchmark":[217],"demonstrate":[218],"effectiveness":[220],"our":[222],"proposed":[223],"methods.":[224]},"counts_by_year":[{"year":2023,"cited_by_count":2}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}