{"id":"https://openalex.org/W4372270459","doi":"https://doi.org/10.1109/icassp49357.2023.10096946","title":"Weighted Sampling for Masked Language Modeling","display_name":"Weighted Sampling for Masked Language Modeling","publication_year":2023,"publication_date":"2023-05-05","ids":{"openalex":"https://openalex.org/W4372270459","doi":"https://doi.org/10.1109/icassp49357.2023.10096946"},"language":"en","primary_location":{"id":"doi:10.1109/icassp49357.2023.10096946","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10096946","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073052283","display_name":"Linhan Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Linhan Zhang","raw_affiliation_strings":["University of New South Wales,School of Computer and Engineering","School of Computer and Engineering, University of New South Wales"],"affiliations":[{"raw_affiliation_string":"University of New South Wales,School of Computer and Engineering","institution_ids":["https://openalex.org/I31746571"]},{"raw_affiliation_string":"School of Computer and Engineering, University of New South Wales","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100428431","display_name":"Qian Chen","orcid":"https://orcid.org/0000-0001-6939-7438"},"institutions":[{"id":"https://openalex.org/I4210086143","display_name":"Alibaba Group (Cayman Islands)","ror":"https://ror.org/00mnrxf72","country_code":"KY","type":"company","lineage":["https://openalex.org/I4210086143","https://openalex.org/I45928872"]}],"countries":["KY"],"is_corresponding":false,"raw_author_name":"Qian Chen","raw_affiliation_strings":["Alibaba Group,Speech Lab of DAMO Academy","Speech Lab of DAMO Academy, Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group,Speech Lab of DAMO Academy","institution_ids":["https://openalex.org/I4210086143"]},{"raw_affiliation_string":"Speech Lab of DAMO Academy, Alibaba Group","institution_ids":["https://openalex.org/I4210086143"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100396375","display_name":"Wen Wang","orcid":"https://orcid.org/0000-0002-0356-1968"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4210086143","display_name":"Alibaba Group (Cayman Islands)","ror":"https://ror.org/00mnrxf72","country_code":"KY","type":"company","lineage":["https://openalex.org/I4210086143","https://openalex.org/I45928872"]}],"countries":["HK","KY"],"is_corresponding":false,"raw_author_name":"Wen Wang","raw_affiliation_strings":["Alibaba Group,Speech Lab of DAMO Academy","Hong Kong University of Science and Technology (Guangzhou),China","Speech Lab of DAMO Academy, Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group,Speech Lab of DAMO Academy","institution_ids":["https://openalex.org/I4210086143"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou),China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Speech Lab of DAMO Academy, Alibaba Group","institution_ids":["https://openalex.org/I4210086143"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041915806","display_name":"Chong Deng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210086143","display_name":"Alibaba Group (Cayman Islands)","ror":"https://ror.org/00mnrxf72","country_code":"KY","type":"company","lineage":["https://openalex.org/I4210086143","https://openalex.org/I45928872"]}],"countries":["KY"],"is_corresponding":false,"raw_author_name":"Chong Deng","raw_affiliation_strings":["Alibaba Group,Speech Lab of DAMO Academy","Speech Lab of DAMO Academy, Alibaba Group"],"affiliations":[{"raw_affiliation_string":"Alibaba Group,Speech Lab of DAMO Academy","institution_ids":["https://openalex.org/I4210086143"]},{"raw_affiliation_string":"Speech Lab of DAMO Academy, Alibaba Group","institution_ids":["https://openalex.org/I4210086143"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5089964993","display_name":"Xin Cao","orcid":"https://orcid.org/0000-0002-3519-7013"},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xin Cao","raw_affiliation_strings":["University of New South Wales,School of Computer and Engineering","School of Computer and Engineering, University of New South Wales"],"affiliations":[{"raw_affiliation_string":"University of New South Wales,School of Computer and Engineering","institution_ids":["https://openalex.org/I31746571"]},{"raw_affiliation_string":"School of Computer and Engineering, University of New South Wales","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073260116","display_name":"Kongzhang Hao","orcid":null},"institutions":[{"id":"https://openalex.org/I31746571","display_name":"UNSW Sydney","ror":"https://ror.org/03r8z3t63","country_code":"AU","type":"education","lineage":["https://openalex.org/I31746571"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Kongzhang Hao","raw_affiliation_strings":["University of New South Wales,School of Computer and Engineering","School of Computer and Engineering, University of New South Wales"],"affiliations":[{"raw_affiliation_string":"University of New South Wales,School of Computer and Engineering","institution_ids":["https://openalex.org/I31746571"]},{"raw_affiliation_string":"School of Computer and Engineering, University of New South Wales","institution_ids":["https://openalex.org/I31746571"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100756716","display_name":"Yuxin Jiang","orcid":"https://orcid.org/0000-0002-8388-7262"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Yuxin Jiang","raw_affiliation_strings":["Hong Kong University of Science and Technology (Guangzhou),China","Hong Kong University of Science and Technology (Guangzhou), China"],"affiliations":[{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou),China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), China","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100391888","display_name":"Wei Wang","orcid":"https://orcid.org/0000-0002-1568-2396"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]},{"id":"https://openalex.org/I4210086143","display_name":"Alibaba Group (Cayman Islands)","ror":"https://ror.org/00mnrxf72","country_code":"KY","type":"company","lineage":["https://openalex.org/I4210086143","https://openalex.org/I45928872"]}],"countries":["HK","KY"],"is_corresponding":false,"raw_author_name":"Wei Wang","raw_affiliation_strings":["Alibaba Group,Speech Lab of DAMO Academy","Hong Kong University of Science and Technology (Guangzhou),China","Hong Kong University of Science and Technology (Guangzhou), China"],"affiliations":[{"raw_affiliation_string":"Alibaba Group,Speech Lab of DAMO Academy","institution_ids":["https://openalex.org/I4210086143"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou),China","institution_ids":["https://openalex.org/I200769079"]},{"raw_affiliation_string":"Hong Kong University of Science and Technology (Guangzhou), China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5073052283"],"corresponding_institution_ids":["https://openalex.org/I31746571"],"apc_list":null,"apc_paid":null,"fwci":0.3579,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.64017413,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":95},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"5"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9736999869346619,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8268009424209595},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7345345616340637},{"id":"https://openalex.org/keywords/security-token","display_name":"Security token","score":0.6717681884765625},{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.670811116695404},{"id":"https://openalex.org/keywords/sentence","display_name":"Sentence","score":0.6448124051094055},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.5955102443695068},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.564818263053894},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5291091799736023},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5187956690788269},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.44688788056373596},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.43968039751052856},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.41033047437667847},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3385632634162903}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8268009424209595},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7345345616340637},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.6717681884765625},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.670811116695404},{"id":"https://openalex.org/C2777530160","wikidata":"https://www.wikidata.org/wiki/Q41796","display_name":"Sentence","level":2,"score":0.6448124051094055},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.5955102443695068},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.564818263053894},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5291091799736023},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5187956690788269},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.44688788056373596},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.43968039751052856},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.41033047437667847},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3385632634162903},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/icassp49357.2023.10096946","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icassp49357.2023.10096946","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-135481","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-135481","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.49000000953674316,"display_name":"Quality Education"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W2126400076","https://openalex.org/W2133458109","https://openalex.org/W2152180407","https://openalex.org/W2250790822","https://openalex.org/W2251861449","https://openalex.org/W2462305634","https://openalex.org/W2739351760","https://openalex.org/W2896457183","https://openalex.org/W2923014074","https://openalex.org/W2945667196","https://openalex.org/W2965373594","https://openalex.org/W2966610483","https://openalex.org/W2973727699","https://openalex.org/W2988217457","https://openalex.org/W2996428491","https://openalex.org/W3104033643","https://openalex.org/W3105816068","https://openalex.org/W3123799706","https://openalex.org/W3131870090","https://openalex.org/W3153427360","https://openalex.org/W3154229486","https://openalex.org/W3156636935","https://openalex.org/W3173777717","https://openalex.org/W3175362188","https://openalex.org/W3213074177","https://openalex.org/W4226441840","https://openalex.org/W4287824654","https://openalex.org/W4292779060","https://openalex.org/W4385573170","https://openalex.org/W4385573468","https://openalex.org/W4385574084","https://openalex.org/W6682338420","https://openalex.org/W6691303741","https://openalex.org/W6691695795","https://openalex.org/W6750615492","https://openalex.org/W6755207826","https://openalex.org/W6757423643","https://openalex.org/W6762715600","https://openalex.org/W6766673545","https://openalex.org/W6767997687","https://openalex.org/W6768021236","https://openalex.org/W6771917389","https://openalex.org/W6778883912","https://openalex.org/W6789902073","https://openalex.org/W6791055906","https://openalex.org/W6794146050","https://openalex.org/W6810587265"],"related_works":["https://openalex.org/W4388335561","https://openalex.org/W2970530566","https://openalex.org/W4288261899","https://openalex.org/W4307309205","https://openalex.org/W2967478618","https://openalex.org/W4385009901","https://openalex.org/W4385572700","https://openalex.org/W2997152889","https://openalex.org/W4387768015","https://openalex.org/W4285141722"],"abstract_inverted_index":{"Masked":[0],"Language":[1],"Modeling":[2],"(MLM)":[3],"is":[4,35],"widely":[5],"used":[6],"to":[7,24,75],"pretrain":[8],"language":[9,21],"models.":[10],"The":[11],"standard":[12],"random":[13],"masking":[14,61],"strategy":[15],"in":[16],"MLM":[17],"causes":[18],"the":[19,84,117,127,132],"pre-trained":[20],"models":[22],"(PLMs)":[23],"be":[25],"biased":[26],"towards":[27],"high-frequency":[28],"tokens.":[29],"Representation":[30],"learning":[31,106,129],"of":[32,131],"rare":[33],"tokens":[34,62],"poor":[36],"and":[37,55,67,77,104,120,138],"PLMs":[38],"have":[39],"limited":[40],"performance":[41],"on":[42,64,83,116],"downstream":[43],"tasks.":[44],"To":[45],"alleviate":[46],"this":[47],"frequency":[48,66],"bias":[49],"issue,":[50],"we":[51],"propose":[52],"two":[53,73],"simple":[54],"effective":[56],"Weighted":[57,123],"Sampling":[58,124],"strategies":[59,74],"for":[60],"based":[63],"token":[65,145],"training":[68],"loss.":[69],"We":[70,111,135],"apply":[71],"these":[72],"BERT":[76,80],"obtain":[78],"Weighted-Sampled":[79],"(WSBERT).":[81],"Experiments":[82],"Semantic":[85],"Textual":[86],"Similarity":[87],"benchmark":[88,119],"(STS)":[89],"show":[90,121],"that":[91,122],"WSBERT":[92,100,115,143],"significantly":[93],"improves":[94,108,126,144],"sentence":[95,109],"embeddings":[96],"over":[97],"BERT.":[98],"Combining":[99],"with":[101],"calibration":[102],"methods":[103],"prompt":[105],"further":[107,136],"embeddings.":[110,146],"also":[112,125],"investigate":[113],"fine-tuning":[114],"GLUE":[118],"transfer":[128],"capability":[130],"backbone":[133],"PLM.":[134],"analyze":[137],"provide":[139],"insights":[140],"into":[141],"how":[142]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":1}],"updated_date":"2026-03-04T09:10:02.777135","created_date":"2025-10-10T00:00:00"}