{"id":"https://openalex.org/W7116303901","doi":"https://doi.org/10.1145/3750720.3757280","title":"Comprehensive Evaluation of LLMs in HPC Code Performance Optimization","display_name":"Comprehensive Evaluation of LLMs in HPC Code Performance Optimization","publication_year":2025,"publication_date":"2025-09-08","ids":{"openalex":"https://openalex.org/W7116303901","doi":"https://doi.org/10.1145/3750720.3757280"},"language":null,"primary_location":{"id":"doi:10.1145/3750720.3757280","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3750720.3757280","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Workshop Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3750720.3757280","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120910204","display_name":"Bowen Cui","orcid":null},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Bowen Cui","raw_affiliation_strings":["Computer Science, George Mason University, Fairfax, Virginia, USA"],"raw_orcid":"https://orcid.org/0009-0004-7261-2548","affiliations":[{"raw_affiliation_string":"Computer Science, George Mason University, Fairfax, Virginia, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120919675","display_name":"Tejas Ramesh","orcid":null},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Tejas Ramesh","raw_affiliation_strings":["Computer Science, George Mason University, Fairfax, Virginia, USA"],"raw_orcid":"https://orcid.org/0009-0008-9522-6518","affiliations":[{"raw_affiliation_string":"Computer Science, George Mason University, Fairfax, Virginia, USA","institution_ids":["https://openalex.org/I162714631"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109854298","display_name":"Oscar R. Hernandez","orcid":null},"institutions":[{"id":"https://openalex.org/I1289243028","display_name":"Oak Ridge National Laboratory","ror":"https://ror.org/01qz5mb56","country_code":"US","type":"facility","lineage":["https://openalex.org/I1289243028","https://openalex.org/I1330989302","https://openalex.org/I39565521","https://openalex.org/I4210159294"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Oscar Hernandez","raw_affiliation_strings":["Oak Ridge National Laboratory, Oak Ridge, Tennessee, USA"],"raw_orcid":"https://orcid.org/0000-0002-5380-6951","affiliations":[{"raw_affiliation_string":"Oak Ridge National Laboratory, Oak Ridge, Tennessee, USA","institution_ids":["https://openalex.org/I1289243028"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063326523","display_name":"Keren Zhou","orcid":"https://orcid.org/0000-0002-7977-3182"},"institutions":[{"id":"https://openalex.org/I162714631","display_name":"George Mason University","ror":"https://ror.org/02jqj7156","country_code":"US","type":"education","lineage":["https://openalex.org/I162714631"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Keren Zhou","raw_affiliation_strings":["Computer Science, George Mason University, Fairfax, Virginia, USA"],"raw_orcid":"https://orcid.org/0000-0002-7977-3182","affiliations":[{"raw_affiliation_string":"Computer Science, George Mason University, Fairfax, Virginia, USA","institution_ids":["https://openalex.org/I162714631"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5120910204"],"corresponding_institution_ids":["https://openalex.org/I162714631"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.64452442,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"1","last_page":"8"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.20419999957084656,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.20419999957084656,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.13050000369548798,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T14347","display_name":"Big Data and Digital Economy","score":0.09570000320672989,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/suite","display_name":"Suite","score":0.7404999732971191},{"id":"https://openalex.org/keywords/workflow","display_name":"Workflow","score":0.6452000141143799},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.640999972820282},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.5867000222206116},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5511999726295471},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5260000228881836}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7491000294685364},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.7404999732971191},{"id":"https://openalex.org/C177212765","wikidata":"https://www.wikidata.org/wiki/Q627335","display_name":"Workflow","level":2,"score":0.6452000141143799},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.640999972820282},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.5867000222206116},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5511999726295471},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5260000228881836},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.44850000739097595},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.40369999408721924},{"id":"https://openalex.org/C139571649","wikidata":"https://www.wikidata.org/wiki/Q1156793","display_name":"Program optimization","level":3,"score":0.37529999017715454},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3240000009536743},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.30799999833106995},{"id":"https://openalex.org/C66024118","wikidata":"https://www.wikidata.org/wiki/Q1122506","display_name":"Computational model","level":2,"score":0.2985999882221222},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C118524514","wikidata":"https://www.wikidata.org/wiki/Q173212","display_name":"Computer architecture","level":1,"score":0.2736999988555908},{"id":"https://openalex.org/C173608175","wikidata":"https://www.wikidata.org/wiki/Q232661","display_name":"Parallel computing","level":1,"score":0.27250000834465027},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.27059999108314514},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.2703000009059906},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.25220000743865967}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3750720.3757280","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3750720.3757280","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Workshop Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3750720.3757280","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3750720.3757280","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Workshop Proceedings of the 54th International Conference on Parallel Processing","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W2080592089","https://openalex.org/W2101778912","https://openalex.org/W2130094715","https://openalex.org/W2134748571","https://openalex.org/W2136434791","https://openalex.org/W2774679324","https://openalex.org/W2987414305","https://openalex.org/W4246166885","https://openalex.org/W4308643319","https://openalex.org/W4313547549","https://openalex.org/W4386730038","https://openalex.org/W4388581500","https://openalex.org/W4393065402","https://openalex.org/W4396790347","https://openalex.org/W4402047561"],"related_works":[],"abstract_inverted_index":{"Large":[0],"Language":[1],"Models":[2],"(LLMs)":[3],"have":[4],"emerged":[5],"as":[6,14,72,74,112],"powerful":[7],"tools":[8],"for":[9,80],"software":[10],"development":[11],"tasks":[12],"such":[13,111],"code":[15,61,151,165],"completion,":[16],"translation,":[17],"and":[18,26,70,116,148,166,173],"optimization.":[19],"However,":[20,153],"their":[21,99,160,167],"ability":[22],"to":[23,56,85,97,162],"generate":[24,163],"efficient":[25],"correct":[27],"code,":[28],"particularly":[29],"in":[30,101,144,169,176],"complex":[31,171],"High-Performance":[32],"Computing":[33],"(HPC)":[34],"contexts,":[35],"has":[36],"remained":[37],"underexplored.":[38],"To":[39],"address":[40],"this":[41,43],"gap,":[42],"paper":[44],"presents":[45],"a":[46,76,92],"comprehensive":[47],"benchmark":[48],"suite":[49],"encompassing":[50],"multiple":[51],"critical":[52],"HPC":[53,81,103,131,178],"computational":[54,88],"motifs":[55],"evaluate":[57],"the":[58,124,136,140],"performance":[59],"of":[60,118,142],"optimized":[62],"by":[63],"state-of-the-art":[64],"LLMs,":[65],"including":[66,159],"OpenAI":[67],"o1,":[68],"Claude-3.5,":[69],"Llama-3.2,":[71],"well":[73],"HPC-Coder,":[75],"model":[77],"tuned":[78],"specifically":[79],"tasks.":[82],"In":[83],"addition":[84],"analyzing":[86],"basic":[87],"kernels,":[89],"we":[90,138,154],"developed":[91],"workflow":[93],"that":[94],"integrates":[95],"LLMs":[96,143],"assess":[98],"effectiveness":[100],"real":[102],"applications.":[104],"Our":[105],"evaluation":[106],"focused":[107],"on":[108,135],"key":[109],"criteria":[110],"execution":[113],"time,":[114],"correctness,":[115],"understanding":[117,145],"HPC-specific":[119],"concepts.":[120],"We":[121],"also":[122,155],"compared":[123],"results":[125],"with":[126],"those":[127],"achieved":[128],"using":[129],"traditional":[130],"optimization":[132],"tools.":[133],"Based":[134],"findings,":[137],"recognized":[139],"strengths":[141],"human":[146],"instructions":[147],"performing":[149],"automated":[150],"transformations.":[152],"identified":[156],"significant":[157],"limitations,":[158],"tendency":[161],"incorrect":[164],"challenges":[168],"comprehending":[170],"control":[172],"data":[174],"flows":[175],"sophisticated":[177],"code.":[179]},"counts_by_year":[],"updated_date":"2025-12-21T23:12:01.093139","created_date":"2025-12-21T00:00:00"}