{"id":"https://openalex.org/W4403536900","doi":"https://doi.org/10.1145/3691620.3695470","title":"JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models","display_name":"JavaBench: A Benchmark of Object-Oriented Code Generation for Evaluating Large Language Models","publication_year":2024,"publication_date":"2024-10-18","ids":{"openalex":"https://openalex.org/W4403536900","doi":"https://doi.org/10.1145/3691620.3695470"},"language":"en","primary_location":{"id":"doi:10.1145/3691620.3695470","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3691620.3695470","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053372458","display_name":"Jialun Cao","orcid":"https://orcid.org/0000-0003-4892-6294"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jialun Cao","raw_affiliation_strings":["The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0003-4892-6294","affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhiyong Chen","orcid":"https://orcid.org/0009-0005-2555-5728"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiyong Chen","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0009-0005-2555-5728","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041677859","display_name":"Jiarong Wu","orcid":"https://orcid.org/0000-0001-6126-303X"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Jiarong Wu","raw_affiliation_strings":["The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0001-6126-303X","affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034057959","display_name":"Shing-Chi Cheung","orcid":"https://orcid.org/0000-0002-3508-7172"},"institutions":[{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["HK"],"is_corresponding":false,"raw_author_name":"Shing-Chi Cheung","raw_affiliation_strings":["The Hong Kong University of Science and Technology, Hong Kong, China"],"raw_orcid":"https://orcid.org/0000-0002-3508-7172","affiliations":[{"raw_affiliation_string":"The Hong Kong University of Science and Technology, Hong Kong, China","institution_ids":["https://openalex.org/I200769079"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5052203980","display_name":"Chang Xu","orcid":"https://orcid.org/0000-0002-6299-4704"},"institutions":[{"id":"https://openalex.org/I881766915","display_name":"Nanjing University","ror":"https://ror.org/01rxvg760","country_code":"CN","type":"education","lineage":["https://openalex.org/I881766915"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chang Xu","raw_affiliation_strings":["Nanjing University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-6299-4704","affiliations":[{"raw_affiliation_string":"Nanjing University, Nanjing, China","institution_ids":["https://openalex.org/I881766915"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":11,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"870","last_page":"882"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7967113256454468},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7462253570556641},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.596973180770874},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5204396843910217},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.5050283074378967},{"id":"https://openalex.org/keywords/object-oriented-programming","display_name":"Object-oriented programming","score":0.43094584345817566},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.10808619856834412},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.060694754123687744},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.056642502546310425}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7967113256454468},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7462253570556641},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.596973180770874},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5204396843910217},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.5050283074378967},{"id":"https://openalex.org/C73752529","wikidata":"https://www.wikidata.org/wiki/Q79872","display_name":"Object-oriented programming","level":2,"score":0.43094584345817566},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.10808619856834412},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.060694754123687744},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.056642502546310425},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1145/3691620.3695470","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3691620.3695470","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 39th IEEE/ACM International Conference on Automated Software Engineering","raw_type":"proceedings-article"},{"id":"pmh:oai:repository.hkust.edu.hk:1783.1-141530","is_oa":false,"landing_page_url":"http://repository.hkust.edu.hk/ir/Record/1783.1-141530","pdf_url":null,"source":{"id":"https://openalex.org/S4306401796","display_name":"Rare & Special e-Zone (The Hong Kong University of Science and Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I200769079","host_organization_name":"Hong Kong University of Science and Technology","host_organization_lineage":["https://openalex.org/I200769079"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Conference paper"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W1990568472","https://openalex.org/W2044802063","https://openalex.org/W2053201696","https://openalex.org/W2101105183","https://openalex.org/W2114728368","https://openalex.org/W2170078167","https://openalex.org/W2561055248","https://openalex.org/W2615762501","https://openalex.org/W2899590559","https://openalex.org/W2951939640","https://openalex.org/W2962936887","https://openalex.org/W2964315653","https://openalex.org/W3098605233","https://openalex.org/W3123182306","https://openalex.org/W3124324659","https://openalex.org/W3159616622","https://openalex.org/W3185341429","https://openalex.org/W3198685994","https://openalex.org/W4237412827","https://openalex.org/W4307479317","https://openalex.org/W4385562549","https://openalex.org/W4385965989","https://openalex.org/W4389104713","https://openalex.org/W4389519225","https://openalex.org/W4389672268","https://openalex.org/W4391876619","https://openalex.org/W4402670434"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W4321353415","https://openalex.org/W1578809901","https://openalex.org/W2163672025","https://openalex.org/W2611825707","https://openalex.org/W2552002334","https://openalex.org/W1547394117","https://openalex.org/W2048831961","https://openalex.org/W1606349578","https://openalex.org/W4399567378"],"abstract_inverted_index":{"Code":[0],"generation":[1],"benchmarks":[2,31,37,57,82,119],"such":[3],"as":[4],"HumanEval":[5],"are":[6,73],"widely":[7],"adopted":[8],"to":[9,48,69,75,120],"evaluate":[10],"LLMs'":[11,46],"capabilities.":[12],"However,":[13],"after":[14],"consolidating":[15],"the":[16,107],"latest":[17],"24":[18],"benchmarks,":[19],"we":[20],"noticed":[21],"three":[22],"significant":[23],"imbalances.":[24],"First,":[25],"imbalanced":[26,53],"programming":[27],"language.":[28],"95.8%":[29],"of":[30,45,62,109],"involve":[32,38],"Python,":[33],"while":[34,94],"only":[35],"5":[36],"Java,":[39],"resulting":[40],"in":[41,113],"an":[42],"insufficient":[43],"understanding":[44],"capability":[47],"generate":[49],"Java":[50,115],"code.":[51],"Second,":[52],"code":[54],"granularity.":[55],"Function-/statement-level":[56],"account":[58],"for":[59],"over":[60],"83.3%":[61],"benchmarks.":[63],"Only":[64],"a":[65],"mere":[66],"handful":[67],"extends":[68],"class-/project-levels,":[70],"and":[71,91,104],"all":[72],"limited":[74],"Python.":[76],"Third,":[77],"lacking":[78],"advanced":[79,96,111],"features.":[80],"Existing":[81],"primarily":[83],"assess":[84],"basic":[85],"coding":[86],"skills":[87],"(e.g.,":[88],"variables,":[89],"operators,":[90],"control":[92],"structures),":[93],"overlooking":[95],"Object-Oriented":[97],"Programming":[98],"(OOP)":[99],"features":[100,112,126],"(i.e.,":[101],"encapsulation,":[102],"inheritance,":[103],"polymorphism).":[105],"Considering":[106],"prevalence":[108],"these":[110],"real-world":[114],"project":[116],"development,":[117],"constructing":[118],"test":[121],"LLMs":[122],"on":[123],"handling":[124],"OOP":[125],"is":[127],"necessary.":[128]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":7}],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2025-10-10T00:00:00"}