{"id":"https://openalex.org/W4394744510","doi":"https://doi.org/10.1145/3597503.3639138","title":"Language Models for Code Completion: A Practical Evaluation","display_name":"Language Models for Code Completion: A Practical Evaluation","publication_year":2024,"publication_date":"2024-04-12","ids":{"openalex":"https://openalex.org/W4394744510","doi":"https://doi.org/10.1145/3597503.3639138"},"language":"en","primary_location":{"id":"doi:10.1145/3597503.3639138","doi":"https://doi.org/10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":{"id":"https://openalex.org/S4210203175","display_name":"ACM/IEEE International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)","issn_l":"0270-5257","issn":["0270-5257","1558-1225"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064355563","display_name":"Maliheh Izadi","orcid":"https://orcid.org/0000-0001-5093-5523","observed_orcids":["https://orcid.org/0000-0001-5093-5523"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Maliheh Izadi","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-5093-5523","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080480598","display_name":"Jonathan Katzy","orcid":"https://orcid.org/0009-0005-9574-2414","observed_orcids":["https://orcid.org/0009-0005-9574-2414"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jonathan Katzy","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0005-9574-2414","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074259780","display_name":"Tim van Dam","orcid":"https://orcid.org/0009-0009-3659-7068","observed_orcids":["https://orcid.org/0009-0009-3659-7068"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tim Van Dam","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0009-3659-7068","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094018467","display_name":"Marc Otten","orcid":"https://orcid.org/0009-0006-6525-8127","observed_orcids":["https://orcid.org/0009-0006-6525-8127"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Marc Otten","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0006-6525-8127","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101986046","display_name":"Razvan Mihai Popescu","orcid":"https://orcid.org/0009-0003-6251-770X","observed_orcids":["https://orcid.org/0009-0003-6251-770X"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Razvan Mihai Popescu","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0003-6251-770X","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090401584","display_name":"Arie Van Deursen A","orcid":"https://orcid.org/0000-0003-4850-3312","observed_orcids":["https://orcid.org/0000-0003-4850-3312"]},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Arie Van Deursen","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-4850-3312","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":20.8136,"has_fulltext":true,"cited_by_count":90,"citation_normalized_percentile":{"value":0.9961456,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-completion","display_name":"code completion","score":0.906000018119812},{"id":"https://openalex.org/keywords/code-language-models","display_name":"code language models","score":0.6639999747276306},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.7419999837875366},{"id":"https://openalex.org/keywords/model-evaluation","display_name":"model evaluation","score":0.6000000238418579},{"id":"https://openalex.org/keywords/programming-languages","display_name":"programming languages","score":0.49300000071525574},{"id":"https://openalex.org/keywords/masking-strategy","display_name":"masking strategy","score":0.902999997138977},{"id":"https://openalex.org/keywords/synthetic-benchmark","display_name":"synthetic benchmark","score":0.4259999990463257}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8375083804130554},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5709680318832397},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5683811902999878},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5480152368545532},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4682266414165497},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4360373020172119},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.424865186214447},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41142675280570984},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40065479278564453},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3834706246852875},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3435802459716797},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.29337939620018005},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3597503.3639138","doi":"https://doi.org/10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":{"id":"https://openalex.org/S4210203175","display_name":"ACM/IEEE International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)","issn_l":"0270-5257","issn":["0270-5257","1558-1225"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2402.16197","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.16197","pdf_url":"https://arxiv.org/pdf/2402.16197","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.16197","doi":"https://doi.org/10.48550/arxiv.2402.16197","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.16197","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1145/3597503.3639138","doi":"https://doi.org/10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":{"id":"https://openalex.org/S4210203175","display_name":"ACM/IEEE International Conference on Software Engineering: Companion Proceedings (ICSE-Companion)","issn_l":"0270-5257","issn":["0270-5257","1558-1225"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"conference"},"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394744510.pdf","grobid_xml":"https://content.openalex.org/works/W4394744510.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W2509810107","https://openalex.org/W2809894362","https://openalex.org/W2892181857","https://openalex.org/W2896457183","https://openalex.org/W2903709763","https://openalex.org/W2903778124","https://openalex.org/W2920812691","https://openalex.org/W2954451301","https://openalex.org/W2965373594","https://openalex.org/W2981852735","https://openalex.org/W3025993830","https://openalex.org/W3030163527","https://openalex.org/W3086007799","https://openalex.org/W3094439486","https://openalex.org/W3098605233","https://openalex.org/W3126675481","https://openalex.org/W3136309015","https://openalex.org/W3138429261","https://openalex.org/W3160765131","https://openalex.org/W3162689995","https://openalex.org/W3177813494","https://openalex.org/W3189653670","https://openalex.org/W3198685994","https://openalex.org/W4212774754","https://openalex.org/W4220672926","https://openalex.org/W4221166942","https://openalex.org/W4224051134","https://openalex.org/W4224060952","https://openalex.org/W4226279206","https://openalex.org/W4226285354","https://openalex.org/W4226485558","https://openalex.org/W4281669078","https://openalex.org/W4281763794","https://openalex.org/W4308648321","https://openalex.org/W4362655426","https://openalex.org/W4375859932","https://openalex.org/W4376167329","https://openalex.org/W4376606797","https://openalex.org/W4384026730","https://openalex.org/W4384345683","https://openalex.org/W4389159031","https://openalex.org/W4389988840","https://openalex.org/W4394638297"],"related_works":["https://openalex.org/W3160765131","https://openalex.org/W7125901482","https://openalex.org/W4384026730","https://openalex.org/W4379919266","https://openalex.org/W2954451301","https://openalex.org/W4405561523","https://openalex.org/W4303711643","https://openalex.org/W4411143671","https://openalex.org/W7125493689","https://openalex.org/W7044142239"],"abstract_inverted_index":{"Transformer-based":[0],"language":[1,35],"models":[2,18,36,78],"for":[3,49,62],"automatic":[4],"code":[5,34],"completion":[6,99],"have":[7],"shown":[8],"great":[9],"promise":[10],"so":[11],"far,":[12],"yet":[13],"the":[14,50,54,103,106,114],"evaluation":[15,52],"of":[16,31,53,96,113],"these":[17],"rarely":[19],"uses":[20],"real":[21,58],"data.":[22],"This":[23],"study":[24,95],"provides":[25],"both":[26],"quantitative":[27],"and":[28,119,129],"qualitative":[29,94],"assessments":[30],"three":[32],"public":[33],"when":[37],"completing":[38],"real-world":[39,98],"code.":[40],"We":[41,56],"first":[42],"developed":[43],"an":[44],"open-source":[45],"IDE":[46],"extension,":[47],"Code4Me,":[48],"online":[51,118],"models.":[55],"collected":[57],"auto-completion":[59],"usage":[60],"data":[61],"over":[63,73],"a":[64,93],"year":[65],"from":[66],"more":[67],"than":[68],"1200":[69],"users,":[70],"resulting":[71],"in":[72,117],"600K":[74],"valid":[75],"completions.":[76],"These":[77],"were":[79],"then":[80],"evaluated":[81],"using":[82,125],"six":[83],"standard":[84],"metrics":[85],"across":[86],"twelve":[87],"programming":[88],"languages.":[89],"Next,":[90],"we":[91],"conducted":[92],"1690":[97],"requests":[100],"to":[101],"identify":[102],"reasons":[104],"behind":[105],"poor":[107],"model":[108],"performance.":[109],"A":[110],"comparative":[111],"analysis":[112],"models'":[115],"performance":[116],"offline":[120],"settings":[121],"was":[122],"also":[123],"performed,":[124],"benchmark":[126],"synthetic":[127],"datasets":[128],"two":[130],"masking":[131],"strategies.":[132]},"counts_by_year":[{"year":2026,"cited_by_count":39},{"year":2025,"cited_by_count":43},{"year":2024,"cited_by_count":8}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2024-04-13T00:00:00"}