{"id":"https://openalex.org/W4402670580","doi":"https://doi.org/10.18653/v1/2024.findings-acl.471","title":"NaturalCodeBench: Examining Coding Performance Mismatch on HumanEval and Natural User Queries","display_name":"NaturalCodeBench: Examining Coding Performance Mismatch on HumanEval and Natural User Queries","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402670580","doi":"https://doi.org/10.18653/v1/2024.findings-acl.471"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-acl.471","doi":"https://doi.org/10.18653/v1/2024.findings-acl.471","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.471","pdf_url":"https://aclanthology.org/2024.findings-acl.471.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.findings-acl.471.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5104152821","display_name":"Shudan Zhang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shudan Zhang","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112655257","display_name":"Han\u2010Lin Zhao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hanlin Zhao","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5029940041","display_name":"Xiao Liu","orcid":"https://orcid.org/0000-0001-6943-9861","observed_orcids":["https://orcid.org/0000-0001-6943-9861"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiao Liu","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065248294","display_name":"Qinkai Zheng","orcid":"https://orcid.org/0000-0002-5391-9446","observed_orcids":["https://orcid.org/0000-0002-5391-9446"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qinkai Zheng","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101384215","display_name":"Zehan Qi","orcid":"https://orcid.org/0009-0007-5232-9130","observed_orcids":["https://orcid.org/0009-0007-5232-9130"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zehan Qi","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102325476","display_name":"Xiaotao Gu","orcid":"https://orcid.org/0009-0002-7896-4750","observed_orcids":["https://orcid.org/0009-0002-7896-4750"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaotao Gu","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026632701","display_name":"Yuxiao Dong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuxiao Dong","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101733791","display_name":"Jie Tang","orcid":"https://orcid.org/0000-0001-5556-2084","observed_orcids":["https://orcid.org/0000-0001-5556-2084"]},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jie Tang","raw_affiliation_strings":["Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":4.1885,"has_fulltext":true,"cited_by_count":11,"citation_normalized_percentile":{"value":0.95176823,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"7907","last_page":"7928"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9696999788284302,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10533","display_name":"Teaching and Learning Programming","score":0.012400000356137753,"subfield":{"id":"https://openalex.org/subfields/1706","display_name":"Computer Science Applications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.007000000216066837,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/humaneval","display_name":"HumanEval","score":0.824999988079071},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.6489999890327454},{"id":"https://openalex.org/keywords/test-case-generation","display_name":"test case generation","score":0.7129999995231628},{"id":"https://openalex.org/keywords/python","display_name":"Python","score":0.3869999945163727},{"id":"https://openalex.org/keywords/java","display_name":"Java","score":0.9980000257492065}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7853437662124634},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4766165316104889},{"id":"https://openalex.org/C2776608160","wikidata":"https://www.wikidata.org/wiki/Q4785462","display_name":"Natural (archaeology)","level":2,"score":0.4459550380706787},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.05483740568161011},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.05227196216583252},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.18653/v1/2024.findings-acl.471","doi":"https://doi.org/10.18653/v1/2024.findings-acl.471","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.471","pdf_url":"https://aclanthology.org/2024.findings-acl.471.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-acl.471","doi":"https://doi.org/10.18653/v1/2024.findings-acl.471","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.471","pdf_url":"https://aclanthology.org/2024.findings-acl.471.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402670580.pdf","grobid_xml":"https://content.openalex.org/works/W4402670580.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4396815387","https://openalex.org/W7162780761","https://openalex.org/W6904016667","https://openalex.org/W4393335400","https://openalex.org/W7125952837","https://openalex.org/W4411119259","https://openalex.org/W7154462824","https://openalex.org/W4405172771","https://openalex.org/W4393903225","https://openalex.org/W7118590070"],"abstract_inverted_index":{"Large":[0],"language":[1],"models":[2,139],"(LLMs)":[3],"have":[4],"manifested":[5],"strong":[6],"ability":[7],"to":[8,56,106,212,236],"generate":[9],"codes":[10],"for":[11,16,97,224],"productive":[12],"activities.However,":[13],"current":[14],"benchmarks":[15],"code":[17,53,155],"synthesis,":[18],"such":[19],"as":[20],"HumanEval,":[21],"MBPP,":[22],"and":[23,33,60,74,179,242],"DS-1000,":[24],"are":[25,182,207],"predominantly":[26],"oriented":[27],"towards":[28],"introductory":[29],"tasks":[30],"on":[31,129,136,153,161,175],"algorithm":[32],"data":[34],"science,":[35],"insufficiently":[36],"satisfying":[37,174],"challenging":[38,52],"requirements":[39],"prevalent":[40],"in":[41,64,72,93,202],"real-world":[42,98],"coding.To":[43],"fill":[44],"this":[45],"gap,":[46],"we":[47,100],"propose":[48],"NATU-RALCODEBENCH":[49],"(NCB),":[50],"a":[51,103,149,221,229],"benchmark":[54],"designed":[55],"mirror":[57],"the":[58,90,108,163,167,237,245],"complexity":[59],"variety":[61],"of":[62,110,122,151,187,190,205,239],"scenarios":[63,157],"real":[65],"coding":[66,84],"tasks.NCB":[67],"comprises":[68],"402":[69],"high-quality":[70],"problems":[71],"Python":[73,222],"Java,":[75],"meticulously":[76],"selected":[77],"from":[78,82,173],"natural":[79],"user":[80],"queries":[81],"online":[83],"services,":[85],"covering":[86],"6":[87],"different":[88],"domains.Noting":[89],"extraordinary":[91],"difficulty":[92],"creating":[94],"testing":[95],"cases":[96],"queries,":[99],"also":[101],"introduce":[102],"semi-automated":[104],"pipeline":[105],"enhance":[107],"efficiency":[109,120],"test":[111],"case":[112],"construction.Comparing":[113],"with":[114,140],"manual":[115],"solutions,":[116],"it":[117],"achieves":[118],"an":[119],"increase":[121],"more":[123],"than":[124,215],"4":[125],"times.Our":[126],"systematic":[127],"experiments":[128],"39":[130],"LLMs":[131],"find":[132],"that":[133],"performance":[134],"gaps":[135],"NCB":[137],"between":[138],"close":[141],"Hu-manEval":[142],"scores":[143],"could":[144],"still":[145,171],"be":[146],"significant,":[147],"indicating":[148],"lack":[150],"focus":[152],"practical":[154],"synthesis":[156],"or":[158],"over-specified":[159],"optimization":[160],"HumanEval.On":[162],"other":[164,214],"hand,":[165],"even":[166],"bestperforming":[168],"GPT-4":[169],"is":[170],"far":[172],"NCB.The":[176],"evaluation":[177],"toolkit":[178],"development":[180],"set":[181],"available":[183],"at":[184],"https://github.":[185],"com/THUDM/NaturalCodeBench.Case":[186],"HumanEval":[188],"Case":[189],"NaturalCodeBench":[191],"def":[192],"has_close_elements(numbers:":[193],"List[float],":[194],"threshold:":[195],"float)":[196],"->":[197],"bool:":[198],"\"\"\"":[199],"Check":[200],"if":[201],"given":[203,216],"list":[204],"numbers,":[206],"any":[208],"two":[209],"numbers":[210],"closer":[211],"each":[213,240],"threshold.\"\"\"":[217],"Hello,":[218],"please":[219],"write":[220],"function":[223,226],"me.The":[225],"should":[227],"read":[228],"markdown":[230],"file,":[231],"add":[232],"numbering":[233],"like":[234],"x.y.z...":[235],"titles":[238],"level,":[241],"then":[243],"return":[244],"modified":[246],"string.":[247]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-26T11:51:55.731654","created_date":"2024-09-21T00:00:00"}