{"id":"https://openalex.org/W3136309015","doi":"https://doi.org/10.1109/msr52588.2021.00024","title":"An Empirical Study on the Usage of BERT Models for Code Completion","display_name":"An Empirical Study on the Usage of BERT Models for Code Completion","alternate_titles":[],"publication_year":2021,"publication_date":"2021-05-01","ids":{"openalex":"https://openalex.org/W3136309015","doi":"https://doi.org/10.1109/msr52588.2021.00024","mag":"3136309015"},"language":"en","primary_location":{"id":"doi:10.1109/msr52588.2021.00024","doi":"https://doi.org/10.1109/msr52588.2021.00024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msr52588.2021.00024","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/ACM 18th International Conference on Mining Software Repositories (MSR)","raw_type":"proceedings-article"},"parent_works":[],"lineage":["https://openalex.org/W3136309015"],"child_works_count":0,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2103.07115","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030004305","display_name":"Matteo Ciniselli","orcid":"https://orcid.org/0000-0002-0251-4473","observed_orcids":["https://orcid.org/0000-0002-0251-4473"]},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Matteo Ciniselli","raw_affiliation_strings":["SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","institution_ids":["https://openalex.org/I57201433"]},{"raw_affiliation_string":"Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland","institution_ids":["https://openalex.org/I57201433"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027300975","display_name":"Nathan Cooper","orcid":"https://orcid.org/0000-0003-2498-705X","observed_orcids":["https://orcid.org/0000-0003-2498-705X"]},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nathan Cooper","raw_affiliation_strings":["William and Mary, USA","William and Mary,SEMERU @ Computer Science Department,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"William and Mary, USA","institution_ids":["https://openalex.org/I16285277"]},{"raw_affiliation_string":"William and Mary,SEMERU @ Computer Science Department,USA","institution_ids":["https://openalex.org/I16285277"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060494552","display_name":"Luca Pascarella","orcid":"https://orcid.org/0000-0001-8289-7736","observed_orcids":["https://orcid.org/0000-0001-8289-7736"]},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Luca Pascarella","raw_affiliation_strings":["SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","institution_ids":["https://openalex.org/I57201433"]},{"raw_affiliation_string":"Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland","institution_ids":["https://openalex.org/I57201433"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041262116","display_name":"Denys Poshyvanyk","orcid":"https://orcid.org/0000-0002-5626-7586","observed_orcids":["https://orcid.org/0000-0002-5626-7586"]},"institutions":[{"id":"https://openalex.org/I16285277","display_name":"William & Mary","ror":"https://ror.org/03hsf0573","country_code":"US","type":"education","lineage":["https://openalex.org/I16285277"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Denys Poshyvanyk","raw_affiliation_strings":["William and Mary, USA","William and Mary,SEMERU @ Computer Science Department,USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"William and Mary, USA","institution_ids":["https://openalex.org/I16285277"]},{"raw_affiliation_string":"William and Mary,SEMERU @ Computer Science Department,USA","institution_ids":["https://openalex.org/I16285277"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025099559","display_name":"Massimiliano Di Penta","orcid":"https://orcid.org/0000-0002-0340-9747","observed_orcids":["https://orcid.org/0000-0002-0340-9747"]},"institutions":[{"id":"https://openalex.org/I16337185","display_name":"University of Sannio","ror":"https://ror.org/04vc81p87","country_code":"IT","type":"education","lineage":["https://openalex.org/I16337185"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Massimiliano Di Penta","raw_affiliation_strings":["University of Sannio, Italy","University of Sannio,Department of Engineering,Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Sannio, Italy","institution_ids":["https://openalex.org/I16337185"]},{"raw_affiliation_string":"University of Sannio,Department of Engineering,Italy","institution_ids":["https://openalex.org/I16337185"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5056526226","display_name":"Gabriele Bavota","orcid":"https://orcid.org/0000-0002-2216-3148","observed_orcids":["https://orcid.org/0000-0002-2216-3148"]},"institutions":[{"id":"https://openalex.org/I57201433","display_name":"Universit\u00e0 della Svizzera italiana","ror":"https://ror.org/03c4atk17","country_code":"CH","type":"education","lineage":["https://openalex.org/I57201433"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Gabriele Bavota","raw_affiliation_strings":["SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SEART @ Software Institute, Universit\u00e0 della Svizzera italiana (USI), Switzerland","institution_ids":["https://openalex.org/I57201433"]},{"raw_affiliation_string":"Universit\u00e0 della Svizzera italiana (USI),SEART @ Software Institute,Switzerland","institution_ids":["https://openalex.org/I57201433"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.7427,"has_fulltext":true,"cited_by_count":126,"citation_normalized_percentile":{"value":0.99176151,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"108","last_page":"119"},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-completion","display_name":"code completion","score":0.9990000128746033},{"id":"https://openalex.org/keywords/bert","display_name":"BERT","score":0.9559999704360962},{"id":"https://openalex.org/keywords/roberta","display_name":"RoBERTa","score":0.9959999918937683},{"id":"https://openalex.org/keywords/integrated-development-environment","display_name":"integrated development environment","score":0.7609999775886536},{"id":"https://openalex.org/keywords/token-prediction","display_name":"token prediction","score":0.6200000047683716},{"id":"https://openalex.org/keywords/bleu","display_name":"BLEU","score":0.5379999876022339},{"id":"https://openalex.org/keywords/levenshtein-distance","display_name":"Levenshtein distance","score":0.9980000257492065}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8195618391036987},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.7026700973510742},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5844516158103943},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4127582907676697},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.40733641386032104},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.36221563816070557},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3475518226623535},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.1490761637687683},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/msr52588.2021.00024","doi":"https://doi.org/10.1109/msr52588.2021.00024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/msr52588.2021.00024","pdf_url":null,"source":{"id":"https://openalex.org/S7407088599","display_name":"International Conference on Mining Software Repositories (MSR)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE/ACM 18th International Conference on Mining Software Repositories (MSR)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2103.07115","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.07115","pdf_url":"https://arxiv.org/pdf/2103.07115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3136309015","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2103.07115.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2103.07115","doi":"https://doi.org/10.48550/arxiv.2103.07115","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2103.07115","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2103.07115","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2103.07115","pdf_url":"https://arxiv.org/pdf/2103.07115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320334678","display_name":"European Research Council","ror":"https://ror.org/0472cxd90"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3136309015.pdf","grobid_xml":"https://content.openalex.org/works/W3136309015.grobid-xml"},"referenced_works_count":61,"referenced_works":["https://openalex.org/W46679369","https://openalex.org/W1647671624","https://openalex.org/W1970018430","https://openalex.org/W1970607969","https://openalex.org/W1977971855","https://openalex.org/W2018389835","https://openalex.org/W2052456234","https://openalex.org/W2059556545","https://openalex.org/W2080579313","https://openalex.org/W2084413241","https://openalex.org/W2096061896","https://openalex.org/W2106259924","https://openalex.org/W2110065044","https://openalex.org/W2115259925","https://openalex.org/W2122076271","https://openalex.org/W2134092629","https://openalex.org/W2142403498","https://openalex.org/W2143861926","https://openalex.org/W2153943889","https://openalex.org/W2157976942","https://openalex.org/W2165747537","https://openalex.org/W2170460608","https://openalex.org/W2266912522","https://openalex.org/W2400997415","https://openalex.org/W2402619042","https://openalex.org/W2478709442","https://openalex.org/W2626778328","https://openalex.org/W2740130862","https://openalex.org/W2797862469","https://openalex.org/W2896457183","https://openalex.org/W2905463374","https://openalex.org/W2907705732","https://openalex.org/W2914161001","https://openalex.org/W2921792613","https://openalex.org/W2954451301","https://openalex.org/W2954823997","https://openalex.org/W2958953787","https://openalex.org/W2963341956","https://openalex.org/W2964322208","https://openalex.org/W2965373594","https://openalex.org/W2980282514","https://openalex.org/W2981852735","https://openalex.org/W3005628256","https://openalex.org/W3011564318","https://openalex.org/W3021206621","https://openalex.org/W3025993830","https://openalex.org/W3034549508","https://openalex.org/W3035300716","https://openalex.org/W3038079173","https://openalex.org/W3092172192","https://openalex.org/W3121707215","https://openalex.org/W3146720657","https://openalex.org/W3150567095","https://openalex.org/W3150708171","https://openalex.org/W3162689995","https://openalex.org/W4232188856","https://openalex.org/W4245415816","https://openalex.org/W4393721841","https://openalex.org/W4394638297","https://openalex.org/W6677035689","https://openalex.org/W6766673545"],"related_works":["https://openalex.org/W3189653670","https://openalex.org/W4414231107","https://openalex.org/W4390187523","https://openalex.org/W4392054274","https://openalex.org/W3015810308","https://openalex.org/W2969101956","https://openalex.org/W4408851842","https://openalex.org/W4416069093","https://openalex.org/W4412807001","https://openalex.org/W4401671212"],"abstract_inverted_index":{"Code":[0],"completion":[1,96],"is":[2,16,30,52],"one":[3,104,191],"of":[4,8,44,58,87,118,132,164,183,232],"the":[5,24,28,41,48,56,59,85,115,133,162,168,180,184,190,218,229,237],"main":[6],"features":[7],"modern":[9],"Integrated":[10],"Development":[11],"Environments":[12],"(IDEs).":[13],"Its":[14],"objective":[15],"to":[17,32,50,55,63,110,189,220,225],"speed":[18],"up":[19,109,224],"code":[20,26,95,112,170,186,207,239],"writing":[21],"by":[22,76,176,193],"predicting":[23],"next":[25,60],"token(s)":[27],"developer":[29],"likely":[31],"write.":[33],"Research":[34],"in":[35,73,93,228],"this":[36,66,74,123],"area":[37],"has":[38],"substantially":[39],"bolstered":[40],"predictive":[42],"performance":[43],"these":[45],"techniques.":[46],"However,":[47],"support":[49],"developers":[51],"still":[53],"limited":[54],"prediction":[57],"few":[61,233],"tokens":[62,234],"type.":[64],"In":[65],"work,":[67],"we":[68,125],"take":[69],"a":[70,78,119,203],"step":[71],"further":[72],"direction":[75],"presenting":[77],"large-scale":[79],"empirical":[80],"study":[81],"aimed":[82],"at":[83,97],"exploring":[84],"capabilities":[86],"state-of-the-art":[88],"deep":[89],"learning":[90],"(DL)":[91],"models":[92,154,201],"supporting":[94],"different":[98],"granularity":[99],"levels,":[100],"including":[101],"single":[102],"tokens,":[103],"or":[105],"multiple":[106],"entire":[107,111,222],"statements,":[108],"blocks":[113],"(e.g.,":[114],"iterated":[116],"block":[117],"for":[120,206],"loop).":[121],"To":[122],"aim,":[124],"train":[126],"and":[127,138,158,178],"test":[128],"several":[129,143],"adapted":[130],"variants":[131],"recently":[134],"proposed":[135],"RoBERTa":[136],"model,":[137],"evaluate":[139],"its":[140],"predictions":[141,166,211],"from":[142,213,236],"perspectives,":[144],"including:":[145],"(i)":[146],"metrics":[147],"usually":[148],"adopted":[149],"when":[150,216],"assessing":[151],"DL":[152],"generative":[153],"(i.e.,":[155,167],"BLEU":[156],"score":[157],"Levenshtein":[159],"distance);":[160],"(ii)":[161],"percentage":[163],"perfect":[165,210],"predicted":[169],"snippets":[171],"that":[172,199],"match":[173],"those":[174],"written":[175,192],"developers);":[177],"(iii)":[179],"\"semantic\"":[181],"equivalence":[182],"generated":[185],"as":[187],"compared":[188],"developers.":[194],"The":[195],"achieved":[196],"results":[197],"show":[198],"BERT":[200],"represent":[202],"viable":[204],"solution":[205],"completion,":[208],"with":[209],"ranging":[212],"~7%,":[214],"obtained":[215],"asking":[217],"model":[219],"guess":[221],"blocks,":[223],"~58%,":[226],"reached":[227],"simpler":[230],"scenario":[231],"masked":[235],"same":[238],"statement.":[240]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3136309015","counts_by_year":[{"year":2026,"cited_by_count":16},{"year":2025,"cited_by_count":26},{"year":2024,"cited_by_count":37},{"year":2023,"cited_by_count":30},{"year":2022,"cited_by_count":12},{"year":2021,"cited_by_count":5}],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2021-03-29T00:00:00"}
