{"id":"https://openalex.org/W7204751395","doi":"https://doi.org/10.57760/sciencedb.010v1","title":"RACE","display_name":"RACE","publication_year":2026,"publication_date":"2026-08-29","ids":{"openalex":"https://openalex.org/W7204751395","doi":"https://doi.org/10.57760/sciencedb.010v1"},"language":"en","primary_location":{"id":"doi:10.57760/sciencedb.010v1","doi":"https://doi.org/10.57760/sciencedb.010v1","is_oa":true,"landing_page_url":"https://doi.org/10.57760/sciencedb.010v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050937","display_name":"ScienceDB","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"volume":null,"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.57760/sciencedb.010v1","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5148823202","display_name":"\u90d1\u5bb6\u80dc","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u90d1\u5bb6\u80dc","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148813746","display_name":"\u66f9\u535a\u5e0c","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u66f9\u535a\u5e0c","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101101329","display_name":"Zhengzhao Ma","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u9a6c\u4e89\u671d","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148773296","display_name":"\u6f58\u82e5\u5f64","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u6f58\u82e5\u5f64","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148772387","display_name":"\u6797\u9e3f\u5b87","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u6797\u9e3f\u5b87","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148764147","display_name":"\u9646\u579a\u6770","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u9646\u579a\u6770","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5148801044","display_name":"\u97e9\u5148\u57f9","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u97e9\u5148\u57f9","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5034536222","display_name":"Le Sun","orcid":"https://orcid.org/0000-0002-4221-0327","observed_orcids":["https://orcid.org/0000-0002-4221-0327","https://orcid.org/0000-0003-2696-7140","https://orcid.org/0000-0002-8750-6295"]},"institutions":[{"id":"https://openalex.org/I4210128818","display_name":"Institute of Software","ror":"https://ror.org/033dfsn42","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210128818"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"\u5b59\u4e50","raw_affiliation_strings":["Institute of Software"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Institute of Software","institution_ids":["https://openalex.org/I4210128818"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210128818"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0007999999797903001,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8700000047683716},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.7710000276565552},{"id":"https://openalex.org/keywords/code-readability","display_name":"code readability","score":0.7379999756813049},{"id":"https://openalex.org/keywords/software-maintainability","display_name":"software maintainability","score":0.968999981880188},{"id":"https://openalex.org/keywords/program-correctness","display_name":"program correctness","score":0.9700000286102295},{"id":"https://openalex.org/keywords/code-efficiency","display_name":"code efficiency","score":0.8659999966621399},{"id":"https://openalex.org/keywords/multidimensional-assessment","display_name":"multidimensional assessment","score":0.6430000066757202},{"id":"https://openalex.org/keywords/data-contamination","display_name":"contaminated data","score":0.6800000071525574},{"id":"https://openalex.org/keywords/software-development-practices","display_name":"software development practices","score":0.6819999814033508}],"concepts":[{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.8148999810218811},{"id":"https://openalex.org/C76509639","wikidata":"https://www.wikidata.org/wiki/Q918036","display_name":"Race (biology)","level":2,"score":0.7159000039100647},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.597599983215332},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4916999936103821},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4593999981880188},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.4569999873638153},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4032000005245209},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.39570000767707825},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3882000148296356},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.33730000257492065},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.325300008058548},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.3181999921798706},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.313400000333786},{"id":"https://openalex.org/C2780586970","wikidata":"https://www.wikidata.org/wiki/Q1357284","display_name":"Popularity","level":2,"score":0.3001999855041504},{"id":"https://openalex.org/C2777212361","wikidata":"https://www.wikidata.org/wiki/Q5127848","display_name":"Class (philosophy)","level":2,"score":0.2572999894618988}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.57760/sciencedb.010v1","doi":"https://doi.org/10.57760/sciencedb.010v1","is_oa":true,"landing_page_url":"https://doi.org/10.57760/sciencedb.010v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050937","display_name":"ScienceDB","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"}],"best_oa_location":{"id":"doi:10.57760/sciencedb.010v1","doi":"https://doi.org/10.57760/sciencedb.010v1","is_oa":true,"landing_page_url":"https://doi.org/10.57760/sciencedb.010v1","pdf_url":null,"source":{"id":"https://openalex.org/S7407050937","display_name":"ScienceDB","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dataset"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4401662005","https://openalex.org/W4415926832","https://openalex.org/W4417170929","https://openalex.org/W4404474721","https://openalex.org/W4417296649","https://openalex.org/W7106804049","https://openalex.org/W4388747891","https://openalex.org/W4392822190","https://openalex.org/W4415622293","https://openalex.org/W7169845512"],"abstract_inverted_index":{"In":[0],"recent":[1],"years,":[2],"researchers":[3],"have":[4],"proposed":[5],"numerous":[6],"benchmarks":[7,21],"to":[8,93,120],"evaluate":[9],"the":[10,24,31,51,114,136,153,161],"impressive":[11],"coding":[12],"capabilities":[13],"of":[14,26,138,155],"large":[15],"language":[16],"models":[17],"(LLMs).":[18],"However,":[19],"current":[20,139],"primarily":[22],"assess":[23],"accuracy":[25],"generated":[27],"code":[28,122,170],"while":[29],"neglecting":[30],"LLMs'":[32,58,169],"multi-dimensional":[33,165],"instruction-following":[34,59,166],"abilities,":[35],"resulting":[36],"in":[37,82],"a":[38,164],"gap":[39],"between":[40],"benchmark":[41],"performance":[42],"and":[43,67,77,96,107],"real-world":[44,177],"development":[45,85],"requirements.":[46],"Therefore,":[47],"this":[48],"paper":[49],"proposes":[50],"RACE":[52,106,128],"benchmark,":[53],"which":[54],"focuses":[55],"on":[56,105],"evaluating":[57],"ability":[60],"across":[61,141],"four":[62],"dimensions:":[63],"Readability,":[64],"mAintainability,":[65],"Correctness,":[66],"Efficiency.":[68],"We":[69,87,99],"select":[70],"specific,":[71],"actionable":[72],"attributes":[73],"for":[74,151,163,168,176],"each":[75],"dimension":[76],"establish":[78],"verifiable":[79],"instructions":[80],"grounded":[81],"common":[83],"software":[84],"practices.":[86],"then":[88],"develop":[89],"deterministic":[90],"evaluation":[91,167],"methods":[92],"ensure":[94],"objective":[95],"reliable":[97],"assessment.":[98],"analyze":[100],"24":[101],"representative":[102],"LLMs":[103,123,140],"based":[104],"find":[108],"that:":[109],"1)":[110],"Using":[111],"correctness":[112,175],"as":[113,147],"only":[115,130],"guiding":[116],"indicator":[117],"is":[118],"insufficient":[119],"steer":[121],"towards":[124],"comprehensive":[125],"advancement.":[126],"2)":[127],"not":[129],"provides":[131],"valuable":[132],"insights":[133],"that":[134],"reveal":[135],"defects":[137],"multiple":[142],"dimensions,":[143],"but":[144],"also":[145],"serves":[146],"an":[148],"effective":[149],"tool":[150],"mitigating":[152],"risks":[154],"data":[156],"contamination.":[157],"These":[158],"findings":[159],"highlight":[160],"need":[162],"ability,":[171],"emphasizing":[172],"metrics":[173],"beyond":[174],"applications.":[178]},"counts_by_year":[],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2026-09-01T00:00:00"}