{"id":"https://openalex.org/W4403536740","doi":"https://doi.org/10.1145/3691620.3695004","title":"A Systematic Evaluation of Large Code Models in API Suggestion: When, Which, and How","display_name":"A Systematic Evaluation of Large Code Models in API Suggestion: When, Which, and How","publication_year":2024,"publication_date":"2024-10-18","ids":{"openalex":"https://openalex.org/W4403536740","doi":"https://doi.org/10.1145/3691620.3695004"},"language":"en","primary_location":{"id":"doi:10.1145/3691620.3695004","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3691620.3695004","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5019372259","display_name":"Chaozheng Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":true,"raw_author_name":"Chaozheng Wang","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012760348","display_name":"Shuzheng Gao","orcid":"https://orcid.org/0000-0002-8102-480X"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Shuzheng Gao","raw_affiliation_strings":["Huawei, Shenzhen, China","The Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Huawei, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061699251","display_name":"Cuiyun Gao","orcid":"https://orcid.org/0000-0003-4774-2434"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cuiyun Gao","raw_affiliation_strings":["Harbin Institute of Technology, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Harbin Institute of Technology, Shenzhen, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5106746626","display_name":"Wenxuan Wang","orcid":"https://orcid.org/0000-0002-9803-8204"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Wenxuan Wang","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024635381","display_name":"Chun Yong Chong","orcid":"https://orcid.org/0000-0003-1164-0049"},"institutions":[{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chun Yong Chong","raw_affiliation_strings":["Huawei, Shenzhen, China"],"affiliations":[{"raw_affiliation_string":"Huawei, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5082429541","display_name":"Shan Gao","orcid":"https://orcid.org/0009-0006-2695-2968"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]},{"id":"https://openalex.org/I2250955327","display_name":"Huawei Technologies (China)","ror":"https://ror.org/00cmhce21","country_code":"CN","type":"company","lineage":["https://openalex.org/I2250955327"]}],"countries":["CN","HK"],"is_corresponding":false,"raw_author_name":"Shan Gao","raw_affiliation_strings":["Huawei, Shenzhen, China","The Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"Huawei, Shenzhen, China","institution_ids":["https://openalex.org/I2250955327"]},{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5069596903","display_name":"Michael R. Lyu","orcid":"https://orcid.org/0000-0002-3666-5798"},"institutions":[{"id":"https://openalex.org/I177725633","display_name":"Chinese University of Hong Kong","ror":"https://ror.org/00t33hh48","country_code":"HK","type":"education","lineage":["https://openalex.org/I177725633"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Michael R. Lyu","raw_affiliation_strings":["The Chinese University of Hong Kong, Hong Kong, China"],"affiliations":[{"raw_affiliation_string":"The Chinese University of Hong Kong, Hong Kong, China","institution_ids":["https://openalex.org/I177725633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":7,"corresponding_author_ids":["https://openalex.org/A5019372259"],"corresponding_institution_ids":["https://openalex.org/I177725633"],"apc_list":null,"apc_paid":null,"fwci":3.1177,"has_fulltext":false,"cited_by_count":4,"citation_normalized_percentile":{"value":0.93141174,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"281","last_page":"293"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12016","display_name":"Web Data Mining and Analysis","score":0.9972000122070312,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6980150938034058},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5577039122581482},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.5338111519813538},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.4310438334941864},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.3753395080566406},{"id":"https://openalex.org/keywords/static-program-analysis","display_name":"Static program analysis","score":0.2493460476398468},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.12763634324073792},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.0731920599937439}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6980150938034058},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5577039122581482},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.5338111519813538},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.4310438334941864},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3753395080566406},{"id":"https://openalex.org/C137287247","wikidata":"https://www.wikidata.org/wiki/Q1329550","display_name":"Static program analysis","level":4,"score":0.2493460476398468},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.12763634324073792},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.0731920599937439},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3691620.3695004","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3691620.3695004","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3616843134","display_name":null,"funder_award_id":"2023A1515011959","funder_id":"https://openalex.org/F4320321921","funder_display_name":"Natural Science Foundation of Guangdong Province"},{"id":"https://openalex.org/G617086500","display_name":null,"funder_award_id":"62472126","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320321921","display_name":"Natural Science Foundation of Guangdong Province","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W2097001189","https://openalex.org/W2402619042","https://openalex.org/W2406365535","https://openalex.org/W2547212960","https://openalex.org/W3005373223","https://openalex.org/W3108032709","https://openalex.org/W3159616622","https://openalex.org/W3176740355","https://openalex.org/W4226075195","https://openalex.org/W4281758439","https://openalex.org/W4284709654","https://openalex.org/W4288055447","https://openalex.org/W4323064995","https://openalex.org/W4387321091","https://openalex.org/W4388483380","https://openalex.org/W4388483492","https://openalex.org/W4388483497","https://openalex.org/W4394769152","https://openalex.org/W4394952072","https://openalex.org/W4399631916"],"related_works":["https://openalex.org/W4389544162","https://openalex.org/W4226317003","https://openalex.org/W2206096527","https://openalex.org/W2149635184","https://openalex.org/W3161903544","https://openalex.org/W4391835699","https://openalex.org/W3017183837","https://openalex.org/W4323034249","https://openalex.org/W4237315075","https://openalex.org/W1969926570"],"abstract_inverted_index":{"API":[0,35,86],"suggestion":[1,36,87],"is":[2],"a":[3,79],"critical":[4],"task":[5,88],"in":[6,25,33,58,89],"modern":[7],"software":[8],"development,":[9],"assisting":[10],"programmers":[11,50],"by":[12],"predicting":[13],"and":[14,67],"recommending":[15],"third-party":[16],"APIs":[17,45,57,66],"based":[18],"on":[19,42],"the":[20,34,64,71,75,85,90],"current":[21],"context.":[22],"Recent":[23],"advancements":[24],"large":[26],"code":[27],"models":[28],"(LCMs)":[29],"have":[30],"shown":[31],"promise":[32],"task.":[37],"However,":[38],"they":[39],"mainly":[40],"focus":[41],"suggesting":[43],"which":[44],"to":[46,62,69],"use,":[47],"ignoring":[48],"that":[49],"may":[51],"demand":[52],"more":[53],"assistance":[54],"while":[55],"using":[56],"practice":[59],"including":[60],"when":[61],"use":[63,70],"suggested":[65],"how":[68],"APIs.":[72],"To":[73],"mitigate":[74],"gap,":[76],"we":[77],"conduct":[78],"systematic":[80],"evaluation":[81],"of":[82],"LCMs":[83],"for":[84],"paper.":[91]},"counts_by_year":[{"year":2025,"cited_by_count":4}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}