{"id":"https://openalex.org/W4411523125","doi":"https://doi.org/10.1145/3728940","title":"ClassEval-T: Evaluating Large Language Models in Class-Level Code Translation","display_name":"ClassEval-T: Evaluating Large Language Models in Class-Level Code Translation","publication_year":2025,"publication_date":"2025-06-22","ids":{"openalex":"https://openalex.org/W4411523125","doi":"https://doi.org/10.1145/3728940"},"language":"en","primary_location":{"id":"doi:10.1145/3728940","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3728940","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5092681481","display_name":"Pengyu Xue","orcid":"https://orcid.org/0009-0007-3395-9575"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pengyu Xue","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0007-3395-9575","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5095886549","display_name":"Linhao Wu","orcid":"https://orcid.org/0009-0001-7624-156X"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Linhao Wu","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0001-7624-156X","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021693730","display_name":"Zhen Yang","orcid":"https://orcid.org/0000-0003-0670-4538"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhen Yang","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0003-0670-4538","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025803036","display_name":"C. Wang","orcid":"https://orcid.org/0009-0000-4153-6774"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chengyi Wang","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0000-4153-6774","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xiang Li","orcid":"https://orcid.org/0009-0007-3473-8221"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiang Li","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0007-3473-8221","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yuxiang Zhang","orcid":"https://orcid.org/0009-0005-5366-9140"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxiang Zhang","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0005-5366-9140","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100405693","display_name":"Jia Li","orcid":"https://orcid.org/0000-0002-5579-8852"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jia Li","raw_affiliation_strings":["Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5579-8852","affiliations":[{"raw_affiliation_string":"Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068735427","display_name":"Ryan Young Jin","orcid":"https://orcid.org/0009-0006-9253-7906"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ruikai Jin","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0006-9253-7906","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yifei Pei","orcid":"https://orcid.org/0009-0005-1351-0565"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yifei Pei","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0005-1351-0565","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032600760","display_name":"Zhaoyan Shen","orcid":"https://orcid.org/0000-0001-9526-6634"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhaoyan Shen","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0000-0001-9526-6634","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Xiran Lyu","orcid":"https://orcid.org/0009-0002-8861-8907"},"institutions":[{"id":"https://openalex.org/I154099455","display_name":"Shandong University","ror":"https://ror.org/0207yh398","country_code":"CN","type":"education","lineage":["https://openalex.org/I154099455"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiran Lyu","raw_affiliation_strings":["Shandong University, Qingdao, China"],"raw_orcid":"https://orcid.org/0009-0002-8861-8907","affiliations":[{"raw_affiliation_string":"Shandong University, Qingdao, China","institution_ids":["https://openalex.org/I154099455"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5051403641","display_name":"Jacky Keung","orcid":"https://orcid.org/0000-0002-3803-9600"},"institutions":[{"id":"https://openalex.org/I168719708","display_name":"City University of Hong Kong","ror":"https://ror.org/03q8dnn23","country_code":"HK","type":"education","lineage":["https://openalex.org/I168719708"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jacky Wai Keung","raw_affiliation_strings":["City University of Hong Kong, Hong Kong, Hong Kong"],"raw_orcid":"https://orcid.org/0000-0002-3803-9600","affiliations":[{"raw_affiliation_string":"City University of Hong Kong, Hong Kong, Hong Kong","institution_ids":["https://openalex.org/I168719708"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":12,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.7946,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":{"value":0.98197131,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"2","issue":"ISSTA","first_page":"1421","last_page":"1444"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9957000017166138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.989300012588501,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6880757808685303},{"id":"https://openalex.org/keywords/python","display_name":"Python (programming language)","score":0.6096603274345398},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.479289710521698},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4423607587814331},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.42787665128707886},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4216011166572571},{"id":"https://openalex.org/keywords/class","display_name":"Class (philosophy)","score":0.41368788480758667},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.37319228053092957},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.08798456192016602},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07660767436027527},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.07342755794525146}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6880757808685303},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6096603274345398},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.479289710521698},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4423607587814331},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42787665128707886},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4216011166572571},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.41368788480758667},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.37319228053092957},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.08798456192016602},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07660767436027527},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.07342755794525146},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3728940","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3728940","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5128106299","display_name":null,"funder_award_id":"Grant Nos. U24B20149 and 62272271","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1969939902","https://openalex.org/W2133437259","https://openalex.org/W2623399293","https://openalex.org/W3098044990","https://openalex.org/W4226004737","https://openalex.org/W4244800048","https://openalex.org/W4250325350","https://openalex.org/W4286904762","https://openalex.org/W4313547549","https://openalex.org/W4376167329","https://openalex.org/W4384302891","https://openalex.org/W4384345708","https://openalex.org/W4388505012","https://openalex.org/W4391558462","https://openalex.org/W4391558518","https://openalex.org/W4391876619","https://openalex.org/W4393905718","https://openalex.org/W4394745858","https://openalex.org/W4394769102","https://openalex.org/W4400318591","https://openalex.org/W4401996408","https://openalex.org/W4402457214","https://openalex.org/W4402670181","https://openalex.org/W4402670434","https://openalex.org/W4402675603","https://openalex.org/W4404900300","https://openalex.org/W6852746770"],"related_works":["https://openalex.org/W2341492732","https://openalex.org/W3187193180","https://openalex.org/W106542691","https://openalex.org/W1699080303","https://openalex.org/W4297799326","https://openalex.org/W3116064965","https://openalex.org/W2378211422","https://openalex.org/W4287027380","https://openalex.org/W2207495067","https://openalex.org/W1906486629"],"abstract_inverted_index":{"In":[0],"recent":[1,88,169,214],"years,":[2],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"have":[7],"dramatically":[8],"advanced":[9],"the":[10,56,82,136,193,208,220,243],"performance":[11,90,189],"of":[12,59,108,171,210],"automated":[13],"code":[14,32,63,76,93,104,145,164,175,198],"translation,":[15],"making":[16],"their":[17],"computational":[18],"accuracy":[19],"score":[20],"reach":[21],"up":[22],"to":[23,85,134,139,230],"over":[24],"80%":[25],"on":[26,91,182],"many":[27],"previous":[28],"benchmarks.":[29],"However,":[30],"most":[31,194],"samples":[33,64,146],"in":[34,61,177,212,253],"these":[35],"benchmarks":[36],"are":[37,248],"short,":[38],"standalone,":[39],"statement/method-level,":[40],"and":[41,80,117,120,127,141,147,160,166,174,180,201,227,251,259],"algorithmic,":[42],"which":[43],"is":[44,53,96],"not":[45],"aligned":[46],"with":[47,143,192],"practical":[48,110,257],"coding":[49,111],"tasks.":[50],"Therefore,":[51],"it":[52],"still":[54],"unknown":[55],"actual":[57],"capability":[58],"LLMs":[60,170,205],"translating":[62,234],"written":[65],"for":[66,162,223,256],"daily":[67],"development.":[68],"To":[69],"achieve":[70],"this,":[71],"we":[72,152,217],"construct":[73],"a":[74,100,187],"class-level":[75,92,102,163],"translation":[77,155,199,225],"benchmark,":[78,200],"ClassEval-T,":[79],"make":[81],"first":[83],"attempt":[84],"extensively":[86],"assess":[87],"LLMs'":[89,228],"translation.":[94],"ClassEval-T":[95,211],"extended":[97],"from":[98],"ClassEval,":[99],"well-known":[101],"Python":[103],"generation":[105],"benchmark":[106],"consisting":[107],"multiple":[109],"topics,":[112],"such":[113],"as":[114],"database":[115],"operation":[116],"game":[118],"design,":[119],"diverse":[121,178,224],"contextual":[122],"dependencies":[123],"(e.g.,":[124],"fields,":[125],"methods,":[126],"libraries).":[128],"It":[129],"cost":[130],"us":[131],"360":[132],"person-hours":[133],"accomplish":[135],"manual":[137],"migration":[138],"Java":[140],"C++":[142],"complete":[144],"associated":[148],"test":[149,247],"suites.":[150],"Subsequently,":[151],"design":[153],"three":[154],"strategies":[156,226],"(i.e.,":[157],"holistic,":[158],"min-dependency,":[159],"standalone)":[161],"translations":[165],"evaluate":[167],"eight":[168],"commercial,":[172],"general,":[173],"kinds":[176],"families":[179],"sizes":[181],"ClassEval-T.":[183],"Experimental":[184],"results":[185],"demonstrate":[186],"remarkable":[188],"drop":[190],"compared":[191],"widely":[195],"studied":[196],"method-level":[197],"obvious":[202],"discrepancies":[203],"among":[204],"appear,":[206],"showing":[207],"effectiveness":[209],"measuring":[213],"LLMs.":[215],"Afterwards,":[216],"further":[218],"discuss":[219],"usage":[221],"scenarios":[222],"ability":[229],"dependency":[231],"awareness":[232],"when":[233],"class":[235],"samples.":[236],"Finally,":[237],"1,243":[238],"failure":[239],"cases":[240],"made":[241],"by":[242],"best-performing":[244],"LLM":[245],"under":[246],"thoroughly":[249],"analyzed":[250],"categorized":[252],"this":[254],"paper":[255],"guidance":[258],"future":[260],"enlightenment.":[261]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":4}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}