{"id":"https://openalex.org/W7133561024","doi":"https://doi.org/10.48550/arxiv.2603.02236","title":"CUDABench: Benchmarking LLMs for Text-to-CUDA Generation","display_name":"CUDABench: Benchmarking LLMs for Text-to-CUDA Generation","publication_year":2026,"publication_date":"2026-02-13","ids":{"openalex":"https://openalex.org/W7133561024","doi":"https://doi.org/10.48550/arxiv.2603.02236"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.02236","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02236","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.02236","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107958486","display_name":"Jiace Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Zhu, Jiace","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121591326","display_name":"Wentao Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Wentao","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128114282","display_name":"Qi Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Qi","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128089694","display_name":"Zhixing Ren","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ren, Zhixing","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128066724","display_name":"Junying Wu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Junying","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128056271","display_name":"Xing Zhe Chai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chai, Xing Zhe","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128104349","display_name":"Chotiwit Rungrueangwutthinon","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rungrueangwutthinon, Chotiwit","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128093242","display_name":"Yehan Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Yehan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5121563724","display_name":"An Zou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zou, An","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":9,"corresponding_author_ids":["https://openalex.org/A5107958486"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.30970001220703125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.30970001220703125,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.08449999988079071,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.0763000026345253,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9139000177383423},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8342999815940857},{"id":"https://openalex.org/keywords/consistency","display_name":"Consistency (knowledge bases)","score":0.7472000122070312},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.6089000105857849},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.550000011920929},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.45989999175071716}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9139000177383423},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8342999815940857},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.7472000122070312},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7260000109672546},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.6089000105857849},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.550000011920929},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.45989999175071716},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4424000084400177},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4235000014305115},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4162999987602234},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.39579999446868896},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38190001249313354},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.3540000021457672},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.30640000104904175},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.29420000314712524},{"id":"https://openalex.org/C149364088","wikidata":"https://www.wikidata.org/wiki/Q185917","display_name":"Translation (biology)","level":4,"score":0.2727000117301941},{"id":"https://openalex.org/C39890363","wikidata":"https://www.wikidata.org/wiki/Q36108","display_name":"Generative grammar","level":2,"score":0.2639000117778778},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.262800008058548}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.02236","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02236","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.02236","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02236","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"studies":[1],"have":[2],"demonstrated":[3],"the":[4,19,27,38,48,68],"potential":[5],"of":[6,21,33,43,50,71,129,147,154],"Large":[7],"Language":[8],"Models":[9],"(LLMs)":[10],"in":[11,82],"generating":[12],"GPU":[13,44,52,155],"Kernels.":[14],"Current":[15],"benchmarks":[16],"focus":[17],"on":[18],"translation":[20],"high-level":[22],"languages":[23],"into":[24],"CUDA,":[25],"overlooking":[26],"more":[28],"general":[29],"and":[30,40,91,99,114,127,141,151],"challenging":[31],"task":[32],"text-to-CUDA":[34,69],"generation.":[35],"Furthermore,":[36,95],"given":[37],"hardware-specific":[39],"performance-critical":[41],"features":[42],"programming,":[45],"accurately":[46],"assessing":[47],"performance":[49],"LLM-generated":[51],"programs":[53],"is":[54,160],"nontrivial.":[55],"In":[56],"this":[57],"work,":[58],"we":[59,74,96],"introduce":[60],"CUDABench,":[61],"a":[62,116,133,145],"comprehensive":[63],"benchmark":[64,159],"designed":[65],"to":[66],"evaluate":[67],"capabilities":[70],"LLMs.":[72],"First,":[73],"construct":[75],"CUDABench-Set,":[76],"which":[77],"covers":[78],"Breadth-Depth-Difficulty":[79],"evaluation":[80],"space":[81],"diverse":[83],"application":[84],"domains,":[85],"including":[86],"artificial":[87],"intelligence,":[88],"scientific":[89],"computing,":[90],"data":[92],"analytics,":[93],"etc.":[94],"propose":[97],"CUDABench-Score":[98],"Generative":[100],"Verification":[101],"Pipeline":[102],"that":[103],"assess":[104],"(1)":[105],"compilation":[106,138],"correctness,":[107,144],"(2)":[108],"functional":[109,143],"consistency":[110],"through":[111],"execution-based":[112],"verification,":[113],"(3)":[115],"novel":[117],"roofline-based":[118],"metric,":[119],"Performance-Score.":[120],"Benchmarking":[121],"state-of-the-art":[122],"LLMs":[123],"reveals":[124],"insightful":[125],"findings":[126],"challenges":[128],"text-to-CUDA,":[130],"such":[131],"as":[132],"notable":[134],"mismatch":[135],"between":[136],"high":[137],"success":[139],"rates":[140],"low":[142],"lack":[146],"domain-specific":[148],"algorithmic":[149],"knowledge,":[150],"suboptimal":[152],"utilization":[153],"hardware":[156],"resources.":[157],"Our":[158],"available":[161],"at":[162],"https://github.com/CUDA-Bench/CUDABench.":[163]},"counts_by_year":[],"updated_date":"2026-03-05T07:36:02.291473","created_date":"2026-03-05T00:00:00"}