{"id":"https://openalex.org/W2954451301","doi":"https://doi.org/10.1109/icse.2019.00101","title":"When Code Completion Fails: A Case Study on Real-World Completions","display_name":"When Code Completion Fails: A Case Study on Real-World Completions","publication_year":2019,"publication_date":"2019-05-01","ids":{"openalex":"https://openalex.org/W2954451301","doi":"https://doi.org/10.1109/icse.2019.00101","mag":"2954451301"},"language":"en","primary_location":{"id":"doi:10.1109/icse.2019.00101","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse.2019.00101","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE/ACM 41st International Conference on Software Engineering (ICSE)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5009679905","display_name":"Vincent J. Hellendoorn","orcid":"https://orcid.org/0000-0001-7516-0525"},"institutions":[{"id":"https://openalex.org/I84218800","display_name":"University of California, Davis","ror":"https://ror.org/05rrcem69","country_code":"US","type":"education","lineage":["https://openalex.org/I84218800"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Vincent J. Hellendoorn","raw_affiliation_strings":["Department of Computer Science, UC Davis Davis, USA"],"affiliations":[{"raw_affiliation_string":"Department of Computer Science, UC Davis Davis, USA","institution_ids":["https://openalex.org/I84218800"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003806730","display_name":"Sebastian Proksch","orcid":"https://orcid.org/0000-0003-1856-9361"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Sebastian Proksch","raw_affiliation_strings":["Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009436227","display_name":"Harald C. Gall","orcid":"https://orcid.org/0000-0002-3874-5628"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Harald C. Gall","raw_affiliation_strings":["Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5082720005","display_name":"Alberto Bacchelli","orcid":"https://orcid.org/0000-0003-0193-6823"},"institutions":[{"id":"https://openalex.org/I202697423","display_name":"University of Zurich","ror":"https://ror.org/02crff812","country_code":"CH","type":"education","lineage":["https://openalex.org/I202697423"]}],"countries":["CH"],"is_corresponding":false,"raw_author_name":"Alberto Bacchelli","raw_affiliation_strings":["Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland"],"affiliations":[{"raw_affiliation_string":"Department of Informatics, University of Zurich, Z\u00fcrich, Switzerland","institution_ids":["https://openalex.org/I202697423"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5009679905"],"corresponding_institution_ids":["https://openalex.org/I84218800"],"apc_list":null,"apc_paid":null,"fwci":14.244,"has_fulltext":false,"cited_by_count":77,"citation_normalized_percentile":{"value":0.98850767,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"960","last_page":"970"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12423","display_name":"Software Reliability and Analysis Research","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7233482003211975},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.5904254913330078},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5847176909446716},{"id":"https://openalex.org/keywords/real-world-data","display_name":"Real world data","score":0.5285564661026001},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.4665061831474304},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.44154107570648193},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4210038185119629},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.30701926350593567},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.26075369119644165},{"id":"https://openalex.org/keywords/static-program-analysis","display_name":"Static program analysis","score":0.2268446981906891},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.1951163411140442}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7233482003211975},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.5904254913330078},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5847176909446716},{"id":"https://openalex.org/C3020493868","wikidata":"https://www.wikidata.org/wiki/Q55631277","display_name":"Real world data","level":2,"score":0.5285564661026001},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.4665061831474304},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.44154107570648193},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4210038185119629},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.30701926350593567},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.26075369119644165},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.2268446981906891},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.1951163411140442}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/icse.2019.00101","is_oa":false,"landing_page_url":"https://doi.org/10.1109/icse.2019.00101","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2019 IEEE/ACM 41st International Conference on Software Engineering (ICSE)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.49000000953674316,"display_name":"Industry, innovation and infrastructure","id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320924","display_name":"Schweizerischer Nationalfonds zur F\u00f6rderung der Wissenschaftlichen Forschung","ror":"https://ror.org/00yjd3n13"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W1771830246","https://openalex.org/W1970607969","https://openalex.org/W1974020522","https://openalex.org/W1994573369","https://openalex.org/W2000897699","https://openalex.org/W2007431958","https://openalex.org/W2018389835","https://openalex.org/W2068050408","https://openalex.org/W2083878868","https://openalex.org/W2118434577","https://openalex.org/W2142403498","https://openalex.org/W2143861926","https://openalex.org/W2145373440","https://openalex.org/W2165747537","https://openalex.org/W2166597811","https://openalex.org/W2266912522","https://openalex.org/W2405053756","https://openalex.org/W2444132761","https://openalex.org/W2463375504","https://openalex.org/W2509810107","https://openalex.org/W2548165777","https://openalex.org/W2602795167","https://openalex.org/W2740130862","https://openalex.org/W2741371834","https://openalex.org/W2771971885","https://openalex.org/W2884670025","https://openalex.org/W2962784628","https://openalex.org/W2963324947","https://openalex.org/W3005855585","https://openalex.org/W3146720657","https://openalex.org/W4213053623","https://openalex.org/W4232728046","https://openalex.org/W4245415816","https://openalex.org/W6718692554"],"related_works":["https://openalex.org/W4389544162","https://openalex.org/W4226317003","https://openalex.org/W2476117260","https://openalex.org/W2206096527","https://openalex.org/W2149635184","https://openalex.org/W2770531922","https://openalex.org/W1845068655","https://openalex.org/W4321354308","https://openalex.org/W3118244000","https://openalex.org/W2290299503"],"abstract_inverted_index":{"Code":[0],"completion":[1,16,35,78,126],"is":[2,9,157],"commonly":[3],"used":[4],"by":[5,93],"software":[6],"developers":[7,180],"and":[8,23,73,100,109,171,197,216,230,233],"integrated":[10],"into":[11],"all":[12],"major":[13],"IDE's.":[14],"Good":[15],"tools":[17,36,110,127],"can":[18],"not":[19],"only":[20],"save":[21],"time":[22],"effort":[24],"but":[25,44],"may":[26],"also":[27],"help":[28],"avoid":[29],"incorrect":[30],"API":[31],"usage.":[32],"Many":[33],"proposed":[34],"have":[37,164],"shown":[38],"promising":[39],"results":[40],"on":[41,86,132,136,206],"synthetic":[42,117,160],"benchmarks,":[43,168],"these":[45],"benchmarks":[46,118],"make":[47],"no":[48],"claims":[49],"about":[50],"the":[51,54,75,137,144,207],"realism":[52],"of":[53,60,70,74,77,122,143],"completions":[55,89,104,139,178],"they":[56],"test.":[57],"This":[58,80],"lack":[59],"grounding":[61],"in":[62,111,159],"real-world":[63,123,133,172,198],"data":[64,222,232],"could":[65],"hinder":[66],"our":[67,227],"scientific":[68],"understanding":[69],"developer":[71,199],"needs":[72],"efficacy":[76],"models.":[79],"paper":[81],"presents":[82],"a":[83],"case":[84],"study":[85,99],"15,000":[87],"code":[88],"that":[90,116,140,156,179],"were":[91,128],"applied":[92],"66":[94],"real":[95],"developers,":[96],"which":[97,211],"we":[98],"contrast":[101],"with":[102],"artificial":[103,221],"to":[105,191,194,203],"inform":[106],"future":[107,167],"research":[108],"this":[112],"area.":[113],"We":[114,224],"find":[115],"misrepresent":[119],"many":[120],"aspects":[121],"completions;":[124],"tested":[125],"far":[129,217],"less":[130,150],"accurate":[131],"data.":[134],"Worse,":[135],"few":[138],"consumed":[141],"most":[142,214],"developers'":[145],"time,":[146],"prediction":[147],"accuracy":[148],"was":[149],"than":[151,220],"20%":[152],"--":[153],"an":[154],"effect":[155],"invisible":[158],"benchmarks.":[161],"Our":[162],"findings":[163],"ramifications":[165],"for":[166,177],"tool":[169],"design":[170],"efficacy:":[173],"Benchmarks":[174],"must":[175],"account":[176],"use":[181],"most,":[182],"such":[183],"as":[184],"intra-project":[185,195],"APIs;":[186],"models":[187],"should":[188],"be":[189,192],"designed":[190],"amenable":[193],"data;":[196],"trials":[200],"are":[201,212],"essential":[202],"quantifying":[204],"performance":[205],"least":[208],"predictable":[209],"completions,":[210],"both":[213],"time-consuming":[215],"more":[218],"typical":[219],"suggests.":[223],"publicly":[225],"release":[226],"preprint":[228],"[https://doi.org/10.5281/zenodo.2565673]":[229],"replication":[231],"materials":[234],"[https://doi.org/10.5281/zenodo.2562249].":[235]},"counts_by_year":[{"year":2025,"cited_by_count":4},{"year":2024,"cited_by_count":15},{"year":2023,"cited_by_count":16},{"year":2022,"cited_by_count":14},{"year":2021,"cited_by_count":15},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}