{"id":"https://openalex.org/W4394745858","doi":"https://doi.org/10.1145/3597503.3639219","title":"Evaluating Large Language Models in Class-Level Code Generation","display_name":"Evaluating Large Language Models in Class-Level Code Generation","publication_year":2024,"publication_date":"2024-04-12","ids":{"openalex":"https://openalex.org/W4394745858","doi":"https://doi.org/10.1145/3597503.3639219"},"language":"en","primary_location":{"id":"doi:10.1145/3597503.3639219","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3597503.3639219","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060885589","display_name":"Xueying Du","orcid":"https://orcid.org/0009-0005-0004-9183"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Xueying Du","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100697907","display_name":"Mingwei Liu","orcid":"https://orcid.org/0000-0002-3462-997X"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mingwei Liu","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039575081","display_name":"Kaixin Wang","orcid":"https://orcid.org/0009-0002-6414-2174"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kaixin Wang","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087522507","display_name":"Hanlin Wang","orcid":"https://orcid.org/0009-0008-2374-1246"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanlin Wang","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5060758777","display_name":"Liu Jun-wei","orcid":null},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junwei Liu","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026843654","display_name":"Yixuan Chen","orcid":"https://orcid.org/0009-0006-8804-1828"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixuan Chen","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008124792","display_name":"Jiayi Feng","orcid":"https://orcid.org/0009-0000-0258-030X"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiayi Feng","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083750020","display_name":"Chaofeng Sha","orcid":"https://orcid.org/0009-0004-4195-0122"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chaofeng Sha","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101854992","display_name":"Xin Peng","orcid":"https://orcid.org/0000-0003-3376-2581"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xin Peng","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5024354460","display_name":"Yiling Lou","orcid":"https://orcid.org/0000-0002-4066-3365"},"institutions":[{"id":"https://openalex.org/I24943067","display_name":"Fudan University","ror":"https://ror.org/013q1eq08","country_code":"CN","type":"education","lineage":["https://openalex.org/I24943067"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yiling Lou","raw_affiliation_strings":["Fudan University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Fudan University, Shanghai, China","institution_ids":["https://openalex.org/I24943067"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5060885589"],"corresponding_institution_ids":["https://openalex.org/I24943067"],"apc_list":null,"apc_paid":null,"fwci":36.0298,"has_fulltext":false,"cited_by_count":105,"citation_normalized_percentile":{"value":0.99841585,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"13"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/statement","display_name":"Statement (logic)","score":0.6720841526985168},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6626067161560059},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6552086472511292},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.5381262302398682},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.48879334330558777},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.4651219844818115},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.4347558617591858},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.3966984152793884},{"id":"https://openalex.org/keywords/natural-language","display_name":"Natural language","score":0.3964275121688843},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.23570159077644348},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.1929064393043518},{"id":"https://openalex.org/keywords/political-science","display_name":"Political science","score":0.1428661346435547},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.1011384129524231}],"concepts":[{"id":"https://openalex.org/C2777026412","wikidata":"https://www.wikidata.org/wiki/Q2684591","display_name":"Statement (logic)","level":2,"score":0.6720841526985168},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6626067161560059},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6552086472511292},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.5381262302398682},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.48879334330558777},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.4651219844818115},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.4347558617591858},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3966984152793884},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.3964275121688843},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.23570159077644348},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1929064393043518},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.1428661346435547},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.1011384129524231},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3597503.3639219","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3597503.3639219","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the IEEE/ACM 46th International Conference on Software Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education","score":0.5699999928474426}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W2052363833","https://openalex.org/W2109479745","https://openalex.org/W2161763921","https://openalex.org/W2962936887","https://openalex.org/W2964315653","https://openalex.org/W3103702776","https://openalex.org/W3195965509","https://openalex.org/W4226242393","https://openalex.org/W4226287673","https://openalex.org/W4226485558","https://openalex.org/W4281763794","https://openalex.org/W4285294723","https://openalex.org/W4285490465","https://openalex.org/W4288055861","https://openalex.org/W4309804142","https://openalex.org/W4313547549","https://openalex.org/W4315706637","https://openalex.org/W4318902699","https://openalex.org/W4319793302","https://openalex.org/W4320559222","https://openalex.org/W4322718191","https://openalex.org/W4322718246","https://openalex.org/W4361806395","https://openalex.org/W4361866100","https://openalex.org/W4362679230","https://openalex.org/W4367860052","https://openalex.org/W4376122390","https://openalex.org/W4376167329","https://openalex.org/W4376312043","https://openalex.org/W4376653732","https://openalex.org/W4377157938","https://openalex.org/W4379933518","https://openalex.org/W4380353763","https://openalex.org/W4380993527","https://openalex.org/W4383472770","https://openalex.org/W4383473945","https://openalex.org/W4383605161","https://openalex.org/W4383605243","https://openalex.org/W4383988989","https://openalex.org/W4384345748","https://openalex.org/W4385373622","https://openalex.org/W4385571452","https://openalex.org/W4385572142","https://openalex.org/W4385681711","https://openalex.org/W4386185318","https://openalex.org/W4387075390","https://openalex.org/W4388483263","https://openalex.org/W4388555487","https://openalex.org/W4389004572","https://openalex.org/W4389158474","https://openalex.org/W4389519225","https://openalex.org/W4389982956","https://openalex.org/W6929340133"],"related_works":["https://openalex.org/W2152921782","https://openalex.org/W382594479","https://openalex.org/W2470045054","https://openalex.org/W2575772232","https://openalex.org/W2151245229","https://openalex.org/W2140902089","https://openalex.org/W2030298461","https://openalex.org/W1510553545","https://openalex.org/W3020827637","https://openalex.org/W199086061"],"abstract_inverted_index":{"Recently,":[0],"many":[1,16],"large":[2],"language":[3,74],"models":[4],"(LLMs)":[5],"have":[6,18],"been":[7,19],"proposed,":[8],"showing":[9],"advanced":[10],"proficiency":[11],"in":[12,94],"code":[13,25,45,52,62,85],"generation.":[14],"Meanwhile,":[15],"efforts":[17],"dedicated":[20],"to":[21,58],"evaluating":[22],"LLMs":[23,57,92],"on":[24,42,79],"generation":[26,46],"benchmarks":[27],"such":[28],"as":[29],"HumanEval.":[30],"Although":[31],"being":[32],"very":[33],"helpful":[34],"for":[35,70],"comparing":[36],"different":[37],"LLMs,":[38],"existing":[39],"evaluation":[40,77],"focuses":[41,78],"a":[43,65,68],"simple":[44],"scenario":[47],"(i.e.,":[48],"function-level":[49],"or":[50,67],"statement-level":[51],"generation),":[53],"which":[54],"mainly":[55],"asks":[56],"generate":[59],"one":[60],"single":[61],"unit":[63],"(e.g.,":[64],"function":[66],"statement)":[69],"the":[71],"given":[72],"natural":[73],"description.":[75],"Such":[76],"generating":[80],"independent":[81],"and":[82],"often":[83],"small-scale":[84],"units,":[86],"thus":[87],"leaving":[88],"it":[89],"unclear":[90],"how":[91],"perform":[93],"real-world":[95],"software":[96],"development":[97],"scenarios.":[98]},"counts_by_year":[{"year":2026,"cited_by_count":12},{"year":2025,"cited_by_count":86},{"year":2024,"cited_by_count":7}],"updated_date":"2026-04-07T14:57:38.498316","created_date":"2025-10-10T00:00:00"}