{"id":"https://openalex.org/W4285490465","doi":"https://doi.org/10.1145/3533767.3534390","title":"An extensive study on pre-trained models for program understanding and generation","display_name":"An extensive study on pre-trained models for program understanding and generation","publication_year":2022,"publication_date":"2022-07-15","ids":{"openalex":"https://openalex.org/W4285490465","doi":"https://doi.org/10.1145/3533767.3534390"},"language":"en","primary_location":{"id":"doi:10.1145/3533767.3534390","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3533767.3534390","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5010506833","display_name":"Zhengran Zeng","orcid":"https://orcid.org/0009-0009-8422-4522"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Zhengran Zeng","raw_affiliation_strings":["Southern University of Science and Technology, China"],"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5059294724","display_name":"Hanzhuo Tan","orcid":"https://orcid.org/0000-0001-5392-5435"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]},{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Hanzhuo Tan","raw_affiliation_strings":["Southern University of Science and Technology, China / Hong Kong Polytechnic University, China"],"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology, China / Hong Kong Polytechnic University, China","institution_ids":["https://openalex.org/I14243506","https://openalex.org/I3045169105"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100392967","display_name":"Haotian Zhang","orcid":"https://orcid.org/0000-0003-0478-3869"},"institutions":[{"id":"https://openalex.org/I2801745840","display_name":"Kwai Chung Hospital","ror":"https://ror.org/05kz7bw59","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I1294586568","https://openalex.org/I2801745840"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haotian Zhang","raw_affiliation_strings":["Kwai, China"],"affiliations":[{"raw_affiliation_string":"Kwai, China","institution_ids":["https://openalex.org/I2801745840"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100336998","display_name":"Jing Li","orcid":"https://orcid.org/0000-0002-8044-2284"},"institutions":[{"id":"https://openalex.org/I14243506","display_name":"Hong Kong Polytechnic University","ror":"https://ror.org/0030zas98","country_code":"HK","type":"education","lineage":["https://openalex.org/I14243506"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jing Li","raw_affiliation_strings":["Hong Kong Polytechnic University, China"],"affiliations":[{"raw_affiliation_string":"Hong Kong Polytechnic University, China","institution_ids":["https://openalex.org/I14243506"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030322956","display_name":"Yuqun Zhang","orcid":"https://orcid.org/0000-0002-1499-5729"},"institutions":[{"id":"https://openalex.org/I3045169105","display_name":"Southern University of Science and Technology","ror":"https://ror.org/049tv2d57","country_code":"CN","type":"education","lineage":["https://openalex.org/I3045169105"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuqun Zhang","raw_affiliation_strings":["Southern University of Science and Technology, China"],"affiliations":[{"raw_affiliation_string":"Southern University of Science and Technology, China","institution_ids":["https://openalex.org/I3045169105"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5043546718","display_name":"Lingming Zhang","orcid":"https://orcid.org/0000-0001-5175-2702"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Lingming Zhang","raw_affiliation_strings":["University of Illinois at Urbana-Champaign, USA"],"affiliations":[{"raw_affiliation_string":"University of Illinois at Urbana-Champaign, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5010506833"],"corresponding_institution_ids":["https://openalex.org/I3045169105"],"apc_list":null,"apc_paid":null,"fwci":40.7404,"has_fulltext":false,"cited_by_count":134,"citation_normalized_percentile":{"value":0.99849921,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"39","last_page":"51"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9922999739646912,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11241","display_name":"Advanced Malware Detection Techniques","score":0.986299991607666,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8419516086578369},{"id":"https://openalex.org/keywords/replicate","display_name":"Replicate","score":0.7166351675987244},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6387932300567627},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.6117469072341919},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5782067775726318},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5455543398857117},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4553641676902771},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.33458200097084045},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.172414630651474},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.10075259208679199}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8419516086578369},{"id":"https://openalex.org/C2781162219","wikidata":"https://www.wikidata.org/wiki/Q26250693","display_name":"Replicate","level":2,"score":0.7166351675987244},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6387932300567627},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.6117469072341919},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5782067775726318},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5455543398857117},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4553641676902771},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.33458200097084045},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.172414630651474},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.10075259208679199},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3533767.3534390","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3533767.3534390","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 31st ACM SIGSOFT International Symposium on Software Testing and Analysis","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.47999998927116394,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":55,"referenced_works":["https://openalex.org/W132347231","https://openalex.org/W2065053490","https://openalex.org/W2516621648","https://openalex.org/W2533695286","https://openalex.org/W2611669587","https://openalex.org/W2788306232","https://openalex.org/W2851896161","https://openalex.org/W2888307014","https://openalex.org/W2899384793","https://openalex.org/W2958754741","https://openalex.org/W2962936887","https://openalex.org/W2963016445","https://openalex.org/W2963341956","https://openalex.org/W2964150020","https://openalex.org/W2972082064","https://openalex.org/W2981851019","https://openalex.org/W2981852735","https://openalex.org/W3009206850","https://openalex.org/W3011013751","https://openalex.org/W3011411500","https://openalex.org/W3011632945","https://openalex.org/W3033748891","https://openalex.org/W3034976548","https://openalex.org/W3034999214","https://openalex.org/W3035507081","https://openalex.org/W3042956498","https://openalex.org/W3043078865","https://openalex.org/W3043761819","https://openalex.org/W3090608524","https://openalex.org/W3091730360","https://openalex.org/W3093604544","https://openalex.org/W3098281910","https://openalex.org/W3098605233","https://openalex.org/W3101449015","https://openalex.org/W3105604018","https://openalex.org/W3105867435","https://openalex.org/W3106196307","https://openalex.org/W3117433489","https://openalex.org/W3121707215","https://openalex.org/W3122184684","https://openalex.org/W3152269143","https://openalex.org/W3161713900","https://openalex.org/W3162962341","https://openalex.org/W3166095789","https://openalex.org/W3170092793","https://openalex.org/W3178061567","https://openalex.org/W3183962691","https://openalex.org/W3195442242","https://openalex.org/W3198659451","https://openalex.org/W3198685994","https://openalex.org/W3200913691","https://openalex.org/W4288089799","https://openalex.org/W4288092382","https://openalex.org/W4300007223","https://openalex.org/W6769216610"],"related_works":["https://openalex.org/W4254851101","https://openalex.org/W3171007296","https://openalex.org/W22115721","https://openalex.org/W2321234655","https://openalex.org/W2065444835","https://openalex.org/W4394550905","https://openalex.org/W2952773340","https://openalex.org/W2470062578","https://openalex.org/W2981861370","https://openalex.org/W4229503580"],"abstract_inverted_index":{"Automatic":[0],"program":[1,44,262,317],"understanding":[2,45,263,318],"and":[3,13,20,46,70,130,172,221,281,296,319],"generation":[4,47],"techniques":[5,175,260],"could":[6],"significantly":[7,256],"advance":[8],"the":[9,23,83,98,115,125,133,169,184,187,199,210,214,229,238,252,268,292],"productivity":[10],"of":[11,25,43,114,148,186,213,237],"programmers":[12],"have":[14],"been":[15],"widely":[16],"studied":[17],"by":[18,53,190],"academia":[19],"industry.":[21],"Recently,":[22],"advent":[24],"pre-trained":[26,33,50,78,116,151,170,178,188,215,240,253,275,294,314],"paradigm":[27],"enlightens":[28],"researchers":[29],"to":[30,127,162],"develop":[31],"general-purpose":[32],"models":[34,79,117,152,171,189,216,241,254,295,315],"which":[35],"can":[36,60,206,227,242,255,289],"be":[37,61],"applied":[38],"for":[39,110,132,176,271,309,316],"a":[40,107,111,154,284],"broad":[41],"range":[42],"tasks.":[48,264],"Such":[49],"models,":[51],"derived":[52],"self-supervised":[54],"objectives":[55],"on":[56,97,118,157,217,313],"large":[57,155],"unlabelled":[58],"corpora,":[59],"fine-tuned":[62],"in":[63,136,207,231,261],"downstream":[64],"tasks":[65,161,220],"(such":[66],"as":[67,122,124],"code":[68,71,160],"search":[69],"generation)":[72],"with":[73],"minimal":[74],"adaptations.":[75],"Although":[76],"these":[77],"claim":[80],"superiority":[81],"over":[82,153,244],"prior":[84],"techniques,":[85],"they":[86,93],"seldom":[87],"follow":[88],"equivalent":[89],"evaluation":[90],"protocols,":[91],"e.g.,":[92],"are":[94],"hardly":[95],"evaluated":[96,219],"identical":[99],"benchmarks,":[100,223],"tasks,":[101],"or":[102],"settings.":[103],"Consequently,":[104],"there":[105],"is":[106],"pressing":[108],"need":[109],"comprehensive":[112],"study":[113,147,270],"their":[119,164,192,218,232],"effectiveness,":[120],"versatility":[121],"well":[123],"limitations":[126],"provide":[128,305],"implications":[129],"guidance":[131],"future":[134,311],"development":[135],"this":[137,140],"area.":[138],"To":[139],"end,":[141],"we":[142,182,201,205,266,303],"first":[143,269],"perform":[144,267],"an":[145],"extensive":[146],"eight":[149],"open-access":[150],"benchmark":[156],"seven":[158],"representative":[159],"assess":[163],"reproducibility.":[165],"We":[166,248],"further":[167],"compare":[168],"domain-specific":[173],"state-of-the-art":[174,259,293],"validating":[177],"effectiveness.":[179],"At":[180,301],"last,":[181,302],"investigate":[183],"robustness":[185,277],"inspecting":[191],"performance":[193,212,225],"variations":[194],"under":[195],"adversarial":[196,279],"attacks.":[197],"Through":[198],"study,":[200],"find":[202,250,282],"that":[203,251,283],"while":[204],"general":[208],"replicate":[209],"original":[211,233],"adopted":[222],"subtle":[224],"fluctuations":[226],"refute":[228],"findings":[230],"papers.":[234],"Moreover,":[235],"none":[236],"existing":[239],"dominate":[243],"all":[245],"other":[246],"models.":[247],"also":[249,304],"outperform":[257],"non-pre-trained":[258],"Furthermore,":[265],"natural":[272],"language-programming":[273],"language":[274],"model":[276],"via":[278],"attacks":[280],"simple":[285],"random":[286],"attack":[287],"approach":[288],"easily":[290],"fool":[291],"thus":[297],"incur":[298],"security":[299],"issues.":[300],"multiple":[306],"practical":[307],"guidelines":[308],"advancing":[310],"research":[312],"generation.":[320]},"counts_by_year":[{"year":2026,"cited_by_count":6},{"year":2025,"cited_by_count":48},{"year":2024,"cited_by_count":47},{"year":2023,"cited_by_count":32},{"year":2022,"cited_by_count":1}],"updated_date":"2026-03-22T08:09:32.410652","created_date":"2025-10-10T00:00:00"}