{"id":"https://openalex.org/W4403296573","doi":"https://doi.org/10.1162/tacl_a_00705","title":"<tt>L2CEval</tt>: Evaluating Language-to-Code Generation Capabilities of Large Language Models","display_name":"<tt>L2CEval</tt>: Evaluating Language-to-Code Generation Capabilities of Large Language Models","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4403296573","doi":"https://doi.org/10.1162/tacl_a_00705"},"language":"en","primary_location":{"id":"doi:10.1162/tacl_a_00705","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00705","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1162/tacl_a_00705","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087977697","display_name":"Ansong Ni","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ansong Ni","raw_affiliation_strings":["Yale University, USA. ansong.ni@yale.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA. ansong.ni@yale.edu","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078519761","display_name":"Pengcheng Yin","orcid":"https://orcid.org/0000-0003-2739-1032"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Pengcheng Yin","raw_affiliation_strings":["Google DeepMind, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Google DeepMind, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047416722","display_name":"Yilun Zhao","orcid":"https://orcid.org/0000-0002-6812-5120"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yilun Zhao","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002538996","display_name":"Martin Riddell","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Martin Riddell","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090611380","display_name":"Troy Feng","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Troy Feng","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112114832","display_name":"Rui Shen","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Rui Shen","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113014595","display_name":"Stephen Yin","orcid":null},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephen Yin","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100346550","display_name":"Liu Ye","orcid":"https://orcid.org/0000-0002-2830-2049"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ye Liu","raw_affiliation_strings":["Salesforce Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, USA","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048500970","display_name":"Semih Yavuz","orcid":null},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Semih Yavuz","raw_affiliation_strings":["Salesforce Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, USA","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032046813","display_name":"Caiming Xiong","orcid":"https://orcid.org/0000-0003-0349-8628"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Caiming Xiong","raw_affiliation_strings":["Salesforce Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, USA","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5005443526","display_name":"Shafiq Joty","orcid":"https://orcid.org/0000-0002-9222-2641"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shafiq Joty","raw_affiliation_strings":["Salesforce Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, USA","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040821130","display_name":"Yingbo Zhou","orcid":"https://orcid.org/0000-0001-6034-9667"},"institutions":[{"id":"https://openalex.org/I4210155268","display_name":"Salesforce (United States)","ror":"https://ror.org/057315g56","country_code":"US","type":"company","lineage":["https://openalex.org/I4210155268"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yingbo Zhou","raw_affiliation_strings":["Salesforce Research, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Salesforce Research, USA","institution_ids":["https://openalex.org/I4210155268"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5081787254","display_name":"Dragomir Radev","orcid":"https://orcid.org/0000-0001-7830-6489"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dragomir Radev","raw_affiliation_strings":["Yale University, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Yale University, USA","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064858748","display_name":"Arman Cohan","orcid":"https://orcid.org/0000-0002-8954-2724"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arman Cohan","raw_affiliation_strings":["Allen Institute for AI, USA","Yale University, USA. arman.cohan@yale.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"Yale University, USA. arman.cohan@yale.edu","institution_ids":["https://openalex.org/I32971472"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5064858748","display_name":"Arman Cohan","orcid":"https://orcid.org/0000-0002-8954-2724"},"institutions":[{"id":"https://openalex.org/I32971472","display_name":"Yale University","ror":"https://ror.org/03v76x132","country_code":"US","type":"education","lineage":["https://openalex.org/I32971472"]},{"id":"https://openalex.org/I4210140341","display_name":"Allen Institute","ror":"https://ror.org/03cpe7c52","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210140341"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arman Cohan","raw_affiliation_strings":["Allen Institute for AI, USA","Yale University, USA. arman.cohan@yale.edu"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Allen Institute for AI, USA","institution_ids":["https://openalex.org/I4210140341"]},{"raw_affiliation_string":"Yale University, USA. arman.cohan@yale.edu","institution_ids":["https://openalex.org/I32971472"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.612,"has_fulltext":false,"cited_by_count":13,"citation_normalized_percentile":{"value":0.93803832,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"12","issue":null,"first_page":"1311","last_page":"1329"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8754754066467285},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.6048799157142639},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.5110064744949341},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.46305394172668457},{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.41762399673461914},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.40907856822013855},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.32003527879714966},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.09399861097335815},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.07121846079826355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8754754066467285},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.6048799157142639},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5110064744949341},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.46305394172668457},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.41762399673461914},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.40907856822013855},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.32003527879714966},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.09399861097335815},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.07121846079826355},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/tacl_a_00705","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00705","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:2fa4818fefd44f2a97b2ec5e9e80af76","is_oa":true,"landing_page_url":"https://doaj.org/article/2fa4818fefd44f2a97b2ec5e9e80af76","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Transactions of the Association for Computational Linguistics, Vol 12 (2024)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/tacl_a_00705","is_oa":true,"landing_page_url":"https://doi.org/10.1162/tacl_a_00705","pdf_url":null,"source":{"id":"https://openalex.org/S2729999759","display_name":"Transactions of the Association for Computational Linguistics","issn_l":"2307-387X","issn":["2307-387X"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Transactions of the Association for Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":98,"referenced_works":["https://openalex.org/W2145618437","https://openalex.org/W2252136820","https://openalex.org/W2254249950","https://openalex.org/W2605202003","https://openalex.org/W2890431379","https://openalex.org/W2950067395","https://openalex.org/W2962936887","https://openalex.org/W2963617989","https://openalex.org/W2963899988","https://openalex.org/W2964315653","https://openalex.org/W2998180104","https://openalex.org/W3001279689","https://openalex.org/W3034758614","https://openalex.org/W3035231859","https://openalex.org/W3087300239","https://openalex.org/W3098044990","https://openalex.org/W3102020135","https://openalex.org/W3105388824","https://openalex.org/W3126675481","https://openalex.org/W3161457214","https://openalex.org/W3170403598","https://openalex.org/W3177813494","https://openalex.org/W3200325732","https://openalex.org/W4221143046","https://openalex.org/W4224060952","https://openalex.org/W4224308101","https://openalex.org/W4225591000","https://openalex.org/W4226053975","https://openalex.org/W4226278401","https://openalex.org/W4283026156","https://openalex.org/W4285225959","https://openalex.org/W4286892945","https://openalex.org/W4287024925","https://openalex.org/W4294753225","https://openalex.org/W4303649020","https://openalex.org/W4309674289","https://openalex.org/W4311887664","https://openalex.org/W4312691425","https://openalex.org/W4315706637","https://openalex.org/W4320165837","https://openalex.org/W4322718191","https://openalex.org/W4327810433","https://openalex.org/W4361866100","https://openalex.org/W4366204357","https://openalex.org/W4368755703","https://openalex.org/W4375869868","https://openalex.org/W4376167329","https://openalex.org/W4378189609","https://openalex.org/W4380993527","https://openalex.org/W4384918448","https://openalex.org/W4385572707","https://openalex.org/W4385572906","https://openalex.org/W4385572953","https://openalex.org/W4386185625","https://openalex.org/W4387561532","https://openalex.org/W4389519979","https://openalex.org/W4394638297","https://openalex.org/W6629631241","https://openalex.org/W6739651123","https://openalex.org/W6767250233","https://openalex.org/W6768003788","https://openalex.org/W6772383348","https://openalex.org/W6784593087","https://openalex.org/W6794686226","https://openalex.org/W6798182279","https://openalex.org/W6800166007","https://openalex.org/W6803096969","https://openalex.org/W6809646742","https://openalex.org/W6810081322","https://openalex.org/W6810162553","https://openalex.org/W6810220367","https://openalex.org/W6810259726","https://openalex.org/W6810389259","https://openalex.org/W6810738896","https://openalex.org/W6811216112","https://openalex.org/W6838461927","https://openalex.org/W6838842041","https://openalex.org/W6839928859","https://openalex.org/W6842556415","https://openalex.org/W6845478269","https://openalex.org/W6846020362","https://openalex.org/W6846648116","https://openalex.org/W6846739993","https://openalex.org/W6846868093","https://openalex.org/W6847168144","https://openalex.org/W6849324701","https://openalex.org/W6849590751","https://openalex.org/W6849732303","https://openalex.org/W6850625674","https://openalex.org/W6850656743","https://openalex.org/W6850853876","https://openalex.org/W6850982024","https://openalex.org/W6851579256","https://openalex.org/W6851928915","https://openalex.org/W6852746770","https://openalex.org/W6852832533","https://openalex.org/W6854013397","https://openalex.org/W6854866820"],"related_works":["https://openalex.org/W2169518243","https://openalex.org/W4231937131","https://openalex.org/W323219885","https://openalex.org/W2063928587","https://openalex.org/W1487966966","https://openalex.org/W1589342014","https://openalex.org/W2163672025","https://openalex.org/W2048831961","https://openalex.org/W1606349578","https://openalex.org/W4399567378"],"abstract_inverted_index":{"Abstract":[0],"Recently,":[1],"large":[2],"language":[3,22],"models":[4],"(LLMs),":[5],"especially":[6],"those":[7],"that":[8,97],"are":[9],"pretrained":[10],"on":[11,46,79,176],"code,":[12],"have":[13],"demonstrated":[14],"strong":[15],"capabilities":[16,76,140],"in":[17,145],"generating":[18],"programs":[19],"from":[20],"natural":[21],"inputs.":[23],"Despite":[24],"promising":[25],"results,":[26],"there":[27],"is":[28],"a":[29,33,56,69,135],"notable":[30],"lack":[31],"of":[32,36,59,72,77,86,138,143],"comprehensive":[34,136],"evaluation":[35,71,151],"these":[37],"models\u2019":[38],"language-to-code":[39,74,146],"generation":[40,75],"capabilities.":[41],"Existing":[42],"studies":[43],"often":[44],"focus":[45],"specific":[47],"tasks,":[48],"model":[49,104,155],"architectures,":[50],"or":[51],"learning":[52],"paradigms,":[53],"leading":[54],"to":[55,124,158],"fragmented":[57],"understanding":[58,137],"the":[60,73,83,95,139,150,160,177],"overall":[61],"landscape.":[62],"In":[63,114],"this":[64],"work,":[65],"we":[66,116],"present":[67],"L2CEval,":[68],"systematic":[70],"LLMs":[78,144],"7":[80],"tasks":[81,130],"across":[82,128],"domain":[84],"spectrum":[85],"semantic":[87],"parsing,":[88],"math":[89],"reasoning,":[90],"and":[91,110,120,131,141,153],"Python":[92],"programming,":[93],"analyzing":[94],"factors":[96],"potentially":[98],"affect":[99],"their":[100],"performance,":[101],"such":[102],"as":[103],"size,":[105],"pretraining":[106],"data,":[107],"instruction":[108],"tuning,":[109],"different":[111,129],"prompting":[112],"methods.":[113],"addition,":[115],"assess":[117],"confidence":[118],"calibration,":[119],"conduct":[121],"human":[122],"evaluations":[123,168],"identify":[125],"typical":[126],"failures":[127],"models.":[132],"L2CEval":[133],"offers":[134],"limitations":[142],"generation.":[147],"We":[148],"release":[149],"framework1":[152],"all":[154],"outputs,":[156],"hoping":[157],"lay":[159],"groundwork":[161],"for":[162],"further":[163],"future":[164,167],"research.":[165],"All":[166],"(e.g.,":[169],"LLaMA-3,":[170],"StarCoder2,":[171],"etc)":[172],"will":[173],"be":[174],"updated":[175],"project":[178],"website:":[179],"https://l2c-eval.github.io/.":[180]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":3}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}