{"id":"https://openalex.org/W4394744510","doi":"https://doi.org/10.1145/3597503.3639138","title":"Language Models for Code Completion: A Practical Evaluation","display_name":"Language Models for Code Completion: A Practical Evaluation","publication_year":2024,"publication_date":"2024-04-12","ids":{"openalex":"https://openalex.org/W4394744510","doi":"https://doi.org/10.1145/3597503.3639138"},"language":"en","primary_location":{"id":"doi:10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5064355563","display_name":"Maliheh Izadi","orcid":"https://orcid.org/0000-0001-5093-5523"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Maliheh Izadi","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0001-5093-5523","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080480598","display_name":"Jonathan Katzy","orcid":"https://orcid.org/0009-0005-9574-2414"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Jonathan Katzy","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0005-9574-2414","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074259780","display_name":"Tim van Dam","orcid":"https://orcid.org/0009-0009-3659-7068"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tim Van Dam","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0009-3659-7068","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5094018467","display_name":"Marc Otten","orcid":null},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Marc Otten","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0006-6525-8127","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101986046","display_name":"Razvan Mihai Popescu","orcid":"https://orcid.org/0009-0003-6251-770X"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Razvan Mihai Popescu","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0009-0003-6251-770X","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5090401584","display_name":"Arie van Deursen","orcid":"https://orcid.org/0000-0003-4850-3312"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Arie Van Deursen","raw_affiliation_strings":["Delft University of Technology, Delft, Netherlands"],"raw_orcid":"https://orcid.org/0000-0003-4850-3312","affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5064355563"],"corresponding_institution_ids":["https://openalex.org/I98358874"],"apc_list":null,"apc_paid":null,"fwci":37.3001,"has_fulltext":true,"cited_by_count":50,"citation_normalized_percentile":{"value":0.99804698,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9861999750137329,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8375083804130554},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5709680318832397},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5683811902999878},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5480152368545532},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4682266414165497},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.4360373020172119},{"id":"https://openalex.org/keywords/open-source","display_name":"Open source","score":0.424865186214447},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.41142675280570984},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.40065479278564453},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3834706246852875},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3435802459716797},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.29337939620018005}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8375083804130554},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5709680318832397},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5683811902999878},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5480152368545532},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4682266414165497},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.4360373020172119},{"id":"https://openalex.org/C3018397939","wikidata":"https://www.wikidata.org/wiki/Q3644502","display_name":"Open source","level":3,"score":0.424865186214447},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.41142675280570984},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.40065479278564453},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3834706246852875},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3435802459716797},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.29337939620018005},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3597503.3639138","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3597503.3639138","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3597503.3639138","source":null,"license":"cc-by-nc-sa","license_id":"https://openalex.org/licenses/cc-by-nc-sa","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.550000011920929}],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4394744510.pdf","grobid_xml":"https://content.openalex.org/works/W4394744510.grobid-xml"},"referenced_works_count":39,"referenced_works":["https://openalex.org/W2509810107","https://openalex.org/W2809894362","https://openalex.org/W2892181857","https://openalex.org/W2896457183","https://openalex.org/W2903778124","https://openalex.org/W2954451301","https://openalex.org/W2970119519","https://openalex.org/W2979792666","https://openalex.org/W2981852735","https://openalex.org/W3086007799","https://openalex.org/W3098605233","https://openalex.org/W3102418234","https://openalex.org/W3108032709","https://openalex.org/W3126675481","https://openalex.org/W3138429261","https://openalex.org/W3160765131","https://openalex.org/W3162689995","https://openalex.org/W3176740355","https://openalex.org/W3177724167","https://openalex.org/W3198685994","https://openalex.org/W3211801722","https://openalex.org/W4212774754","https://openalex.org/W4220672926","https://openalex.org/W4221166942","https://openalex.org/W4224060952","https://openalex.org/W4226285354","https://openalex.org/W4226485558","https://openalex.org/W4281669078","https://openalex.org/W4281763794","https://openalex.org/W4308648321","https://openalex.org/W4315705963","https://openalex.org/W4319793451","https://openalex.org/W4362655426","https://openalex.org/W4375859932","https://openalex.org/W4376606797","https://openalex.org/W4384026730","https://openalex.org/W4389988840","https://openalex.org/W6778883912","https://openalex.org/W6848604973"],"related_works":["https://openalex.org/W4288365749","https://openalex.org/W2936497627","https://openalex.org/W3013624417","https://openalex.org/W4287826556","https://openalex.org/W2113128227","https://openalex.org/W3098382480","https://openalex.org/W4287598411","https://openalex.org/W3094871513","https://openalex.org/W3100913109","https://openalex.org/W3198458223"],"abstract_inverted_index":{"Transformer-based":[0],"language":[1,35],"models":[2,18,36,78],"for":[3,49,62],"automatic":[4],"code":[5,34],"completion":[6,99],"have":[7],"shown":[8],"great":[9],"promise":[10],"so":[11],"far,":[12],"yet":[13],"the":[14,50,54,103,106,114],"evaluation":[15,52],"of":[16,31,53,96,113],"these":[17],"rarely":[19],"uses":[20],"real":[21,58],"data.":[22],"This":[23],"study":[24,95],"provides":[25],"both":[26],"quantitative":[27],"and":[28,119,129],"qualitative":[29,94],"assessments":[30],"three":[32],"public":[33],"when":[37],"completing":[38],"real-world":[39,98],"code.":[40],"We":[41,56],"first":[42],"developed":[43],"an":[44],"open-source":[45],"IDE":[46],"extension,":[47],"Code4Me,":[48],"online":[51,118],"models.":[55],"collected":[57],"auto-completion":[59],"usage":[60],"data":[61],"over":[63,73],"a":[64,93],"year":[65],"from":[66],"more":[67],"than":[68],"1200":[69],"users,":[70],"resulting":[71],"in":[72,117],"600K":[74],"valid":[75],"completions.":[76],"These":[77],"were":[79],"then":[80],"evaluated":[81],"using":[82,125],"six":[83],"standard":[84],"metrics":[85],"across":[86],"twelve":[87],"programming":[88],"languages.":[89],"Next,":[90],"we":[91],"conducted":[92],"1690":[97],"requests":[100],"to":[101],"identify":[102],"reasons":[104],"behind":[105],"poor":[107],"model":[108],"performance.":[109],"A":[110],"comparative":[111],"analysis":[112],"models'":[115],"performance":[116],"offline":[120],"settings":[121],"was":[122],"also":[123],"performed,":[124],"benchmark":[126],"synthetic":[127],"datasets":[128],"two":[130],"masking":[131],"strategies.":[132]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":39},{"year":2024,"cited_by_count":6}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}