{"id":"https://openalex.org/W4403336760","doi":"https://doi.org/10.1145/3699599","title":"Improving Source Code Pre-Training via Type-Specific Masking","display_name":"Improving Source Code Pre-Training via Type-Specific Masking","publication_year":2024,"publication_date":"2024-10-11","ids":{"openalex":"https://openalex.org/W4403336760","doi":"https://doi.org/10.1145/3699599"},"language":"en","primary_location":{"id":"doi:10.1145/3699599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699599","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699599","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3699599","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104726373","display_name":"Wentao Zou","orcid":null},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wentao Zou","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","National Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0009-0000-3385-7607","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Qi Li","orcid":"https://orcid.org/0009-0002-1495-2697"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qi Li","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","National Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0009-0002-1495-2697","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103039853","display_name":"Chuanyi Li","orcid":"https://orcid.org/0000-0001-9270-5072"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chuanyi Li","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","National Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0001-9270-5072","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075214885","display_name":"Jidong Ge","orcid":"https://orcid.org/0000-0003-1773-0942"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jidong Ge","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","National Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0003-1773-0942","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100441911","display_name":"Xiang Chen","orcid":"https://orcid.org/0000-0002-1180-3891"},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang Chen","raw_affiliation_strings":["School of Artificial Intelligence and Computer Science, Nantong University, Nantong, China","School of Information Science and Technology, Nantong University, China"],"raw_orcid":"https://orcid.org/0000-0002-1180-3891","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence and Computer Science, Nantong University, Nantong, China","institution_ids":["https://openalex.org/I199305430"]},{"raw_affiliation_string":"School of Information Science and Technology, Nantong University, China","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028716858","display_name":"LiGuo Huang","orcid":"https://orcid.org/0000-0001-7790-0195"},"institutions":[{"id":"https://openalex.org/I178169726","display_name":"Southern Methodist University","ror":"https://ror.org/042tdr378","country_code":"US","type":"education","lineage":["https://openalex.org/I178169726"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"LiGuo Huang","raw_affiliation_strings":["Department of Computer Science, Southern Methodist University, Dallas, TX, USA","Department of Computer Science, Southern Methodist University, USA"],"raw_orcid":"https://orcid.org/0000-0001-7790-0195","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Southern Methodist University, Dallas, TX, USA","institution_ids":["https://openalex.org/I178169726"]},{"raw_affiliation_string":"Department of Computer Science, Southern Methodist University, USA","institution_ids":["https://openalex.org/I178169726"]}]},{"author_position":"last","author":{"id":null,"display_name":"Bin Luo","orcid":"https://orcid.org/0000-0002-9036-0063"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Luo","raw_affiliation_strings":["National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","National Key Laboratory for Novel Software Technology, Nanjing University, China"],"raw_orcid":"https://orcid.org/0000-0002-9036-0063","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]},{"raw_affiliation_string":"National Key Laboratory for Novel Software Technology, Nanjing University, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.15052038,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"34","issue":"3","first_page":"1","last_page":"34"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9930999875068665,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8375173807144165},{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.4789050817489624},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.46168938279151917},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2904347777366638}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8375173807144165},{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.4789050817489624},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.46168938279151917},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2904347777366638},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3699599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699599","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699599","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3699599","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3699599","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3699599","source":{"id":"https://openalex.org/S142627899","display_name":"ACM Transactions on Software Engineering and Methodology","issn_l":"1049-331X","issn":["1049-331X","1557-7392"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM Transactions on Software Engineering and Methodology","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4399999976158142,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403336760.pdf"},"referenced_works_count":73,"referenced_works":["https://openalex.org/W1593271688","https://openalex.org/W2065053490","https://openalex.org/W2072750798","https://openalex.org/W2282866165","https://openalex.org/W2753709519","https://openalex.org/W2950813464","https://openalex.org/W2954149564","https://openalex.org/W2962936887","https://openalex.org/W2981852735","https://openalex.org/W3000135256","https://openalex.org/W3000546628","https://openalex.org/W3008088841","https://openalex.org/W3011564318","https://openalex.org/W3014451403","https://openalex.org/W3016234956","https://openalex.org/W3034238904","https://openalex.org/W3034999214","https://openalex.org/W3035010485","https://openalex.org/W3093464584","https://openalex.org/W3096150021","https://openalex.org/W3101498587","https://openalex.org/W3103748122","https://openalex.org/W3108032709","https://openalex.org/W3109716606","https://openalex.org/W3119558324","https://openalex.org/W3121217868","https://openalex.org/W3121707215","https://openalex.org/W3152918650","https://openalex.org/W3159616622","https://openalex.org/W3170962973","https://openalex.org/W3173162544","https://openalex.org/W3183962691","https://openalex.org/W3198685994","https://openalex.org/W3217001695","https://openalex.org/W4205371973","https://openalex.org/W4205596491","https://openalex.org/W4206756860","https://openalex.org/W4221153183","https://openalex.org/W4226057883","https://openalex.org/W4226395508","https://openalex.org/W4280534475","https://openalex.org/W4281763794","https://openalex.org/W4283209692","https://openalex.org/W4284688961","https://openalex.org/W4284696121","https://openalex.org/W4285220366","https://openalex.org/W4285338683","https://openalex.org/W4285490465","https://openalex.org/W4286530321","https://openalex.org/W4288804596","https://openalex.org/W4300113433","https://openalex.org/W4308643994","https://openalex.org/W4312282218","https://openalex.org/W4313547604","https://openalex.org/W4313563649","https://openalex.org/W4313563681","https://openalex.org/W4322766882","https://openalex.org/W4323066747","https://openalex.org/W4330337479","https://openalex.org/W4367595583","https://openalex.org/W4382202657","https://openalex.org/W4384302759","https://openalex.org/W4384345646","https://openalex.org/W4384345698","https://openalex.org/W4384345795","https://openalex.org/W4384918448","https://openalex.org/W4385236934","https://openalex.org/W4386185625","https://openalex.org/W4386302049","https://openalex.org/W4389988584","https://openalex.org/W4391272793","https://openalex.org/W4391766565","https://openalex.org/W6778883912"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052","https://openalex.org/W4402327032","https://openalex.org/W2382290278"],"abstract_inverted_index":{"The":[0],"Masked":[1],"Language":[2],"Modeling":[3],"(MLM)":[4],"task":[5],"is":[6,64],"widely":[7],"recognized":[8],"as":[9,53],"one":[10],"of":[11,30,166,174,192],"the":[12,23,162,172,188],"most":[13,29],"effective":[14],"pre-training":[15,156],"tasks":[16,91,134,157],"and":[17,62,108,116,120,138,164,177,190],"currently":[18],"derives":[19],"many":[20],"variants":[21,32],"in":[22],"Software":[24],"Engineering":[25],"(SE)":[26],"field.":[27],"However,":[28],"these":[31],"mainly":[33],"focus":[34,47],"on":[35,48,123,140,181],"code":[36,42,54,58,99],"representation":[37],"without":[38],"distinguishing":[39],"between":[40],"different":[41,78,88,193],"token":[43,59,100],"types,":[44,79,101],"while":[45],"some":[46],"a":[49],"specific":[50],"type,":[51],"such":[52],"identifiers.":[55],"Indeed,":[56],"various":[57],"types":[60],"exist,":[61],"there":[63],"no":[65],"evidence":[66],"that":[67,131,154,170],"only":[68],"identifiers":[69],"can":[70,92,135],"improve":[71,75],"PTMs.":[72,94,183],"Thus,":[73],"to":[74,85,149],"PTMs":[76,153],"through":[77],"we":[80,96,113,145],"conducted":[81],"an":[82],"extensive":[83],"study":[84],"evaluate":[86],"how":[87],"type-specific":[89,105,132,194],"masking":[90,106,133,195],"affect":[93],"First,":[95],"extract":[97],"five":[98],"convert":[102],"them":[103,122],"into":[104],"tasks,":[107],"generate":[109],"their":[110],"combinations.":[111],"Second,":[112],"pre-train":[114],"CodeBERT":[115,137],"PLBART":[117,139],"using":[118],"combinations":[119],"fine-tuned":[121],"four":[124],"SE":[125],"downstream":[126,142],"tasks.":[127,143,196],"Experimental":[128],"results":[129],"show":[130],"enhance":[136],"all":[141],"Furthermore,":[144],"discuss":[146],"topics":[147],"related":[148],"low-resource":[150],"datasets,":[151],"conflicting":[152],"original":[155],"conflict":[158],"with":[159],"our":[160,167,175,179],"methods,":[161,168,176],"cost":[163],"performance":[165,173],"factors":[169],"impact":[171],"applying":[178],"methods":[180],"state-of-the-art":[182],"These":[184],"discussions":[185],"comprehensively":[186],"analyze":[187],"strengths":[189],"weaknesses":[191]},"counts_by_year":[],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}