{"id":"https://openalex.org/W4388561281","doi":"https://doi.org/10.1145/3624062.3624135","title":"Towards Collaborative Continuous Benchmarking for HPC","display_name":"Towards Collaborative Continuous Benchmarking for HPC","publication_year":2023,"publication_date":"2023-11-10","ids":{"openalex":"https://openalex.org/W4388561281","doi":"https://doi.org/10.1145/3624062.3624135"},"language":"en","primary_location":{"id":"doi:10.1145/3624062.3624135","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3624062.3624135","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3624062.3624135","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '23 Workshops of the International Conference on High Performance Computing, Network, Storage, and Analysis","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3624062.3624135","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5012779678","display_name":"Olga Pearce","orcid":"https://orcid.org/0000-0002-1904-9627"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Olga Pearce","raw_affiliation_strings":["Lawrence Livermore National Laboratory, United States of America and Texas A&M University, United States of America"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, United States of America and Texas A&M University, United States of America","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021237429","display_name":"A.. Scott","orcid":"https://orcid.org/0000-0001-6255-1308"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alec Scott","raw_affiliation_strings":["Lawrence Livermore National Laboratory, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, USA","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021153793","display_name":"Gregory Becker","orcid":"https://orcid.org/0000-0002-8472-4449"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gregory Becker","raw_affiliation_strings":["Lawrence Livermore National Laboratory, United States of America"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, United States of America","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102715906","display_name":"Riyaz Haque","orcid":"https://orcid.org/0000-0001-8930-5721"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Riyaz Haque","raw_affiliation_strings":["Lawrence Livermore National Laboratory, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, USA","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5024635881","display_name":"Nathan Hanford","orcid":"https://orcid.org/0000-0002-2214-7447"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nathan Hanford","raw_affiliation_strings":["Lawrence Livermore National Laboratory, United States of America"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, United States of America","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010860138","display_name":"Stephanie Brink","orcid":"https://orcid.org/0000-0002-1458-8453"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stephanie Brink","raw_affiliation_strings":["Lawrence Livermore National Laboratory, United States of America"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, United States of America","institution_ids":["https://openalex.org/I1282311441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010589863","display_name":"Douglas W. Jacobsen","orcid":"https://orcid.org/0000-0002-3836-207X"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Doug Jacobsen","raw_affiliation_strings":["Google, United States of America"],"affiliations":[{"raw_affiliation_string":"Google, United States of America","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075074101","display_name":"Heidi Poxon","orcid":"https://orcid.org/0009-0005-4104-3968"},"institutions":[{"id":"https://openalex.org/I1311688040","display_name":"Amazon (United States)","ror":"https://ror.org/04mv4n011","country_code":"US","type":"company","lineage":["https://openalex.org/I1311688040"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Heidi Poxon","raw_affiliation_strings":["Amazon, USA"],"affiliations":[{"raw_affiliation_string":"Amazon, USA","institution_ids":["https://openalex.org/I1311688040"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039069961","display_name":"Jens Domke","orcid":"https://orcid.org/0000-0002-5343-414X"},"institutions":[{"id":"https://openalex.org/I4210110652","display_name":"RIKEN","ror":"https://ror.org/01sjwvz98","country_code":"JP","type":"facility","lineage":["https://openalex.org/I4210110652"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Jens Domke","raw_affiliation_strings":["RIKEN, Japan"],"affiliations":[{"raw_affiliation_string":"RIKEN, Japan","institution_ids":["https://openalex.org/I4210110652"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007709486","display_name":"Todd Gamblin","orcid":"https://orcid.org/0000-0002-7857-2805"},"institutions":[{"id":"https://openalex.org/I1282311441","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53","country_code":"US","type":"facility","lineage":["https://openalex.org/I1282311441","https://openalex.org/I1330989302","https://openalex.org/I198811213","https://openalex.org/I4210138311"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Todd Gamblin","raw_affiliation_strings":["Lawrence Livermore National Laboratory, USA"],"affiliations":[{"raw_affiliation_string":"Lawrence Livermore National Laboratory, USA","institution_ids":["https://openalex.org/I1282311441"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5012779678"],"corresponding_institution_ids":["https://openalex.org/I1282311441"],"apc_list":null,"apc_paid":null,"fwci":2.596,"has_fulltext":true,"cited_by_count":13,"citation_normalized_percentile":{"value":0.90458834,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"627","last_page":"635"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11181","display_name":"Advanced Data Storage Technologies","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10715","display_name":"Distributed and Parallel Computing Systems","score":0.996999979019165,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.9903525114059448},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7330330610275269},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.7217119932174683},{"id":"https://openalex.org/keywords/automation","display_name":"Automation","score":0.4746837317943573},{"id":"https://openalex.org/keywords/supercomputer","display_name":"Supercomputer","score":0.4699654281139374},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.37437450885772705},{"id":"https://openalex.org/keywords/process-management","display_name":"Process management","score":0.3421466052532196},{"id":"https://openalex.org/keywords/engineering-management","display_name":"Engineering management","score":0.341350257396698},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.3049337863922119},{"id":"https://openalex.org/keywords/embedded-system","display_name":"Embedded system","score":0.18895015120506287},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1465693712234497},{"id":"https://openalex.org/keywords/business","display_name":"Business","score":0.09049469232559204}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.9903525114059448},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7330330610275269},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.7217119932174683},{"id":"https://openalex.org/C115901376","wikidata":"https://www.wikidata.org/wiki/Q184199","display_name":"Automation","level":2,"score":0.4746837317943573},{"id":"https://openalex.org/C83283714","wikidata":"https://www.wikidata.org/wiki/Q121117","display_name":"Supercomputer","level":2,"score":0.4699654281139374},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.37437450885772705},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.3421466052532196},{"id":"https://openalex.org/C110354214","wikidata":"https://www.wikidata.org/wiki/Q6314146","display_name":"Engineering management","level":1,"score":0.341350257396698},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.3049337863922119},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.18895015120506287},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1465693712234497},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.09049469232559204},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3624062.3624135","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3624062.3624135","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3624062.3624135","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '23 Workshops of the International Conference on High Performance Computing, Network, Storage, and Analysis","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3624062.3624135","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3624062.3624135","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3624062.3624135","source":null,"license":"cc-by-nd","license_id":"https://openalex.org/licenses/cc-by-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the SC '23 Workshops of the International Conference on High Performance Computing, Network, Storage, and Analysis","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1494814522","display_name":null,"funder_award_id":"DE_AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G2917824471","display_name":null,"funder_award_id":"DE-AC52-07NA2734","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G3391580208","display_name":null,"funder_award_id":"DE-AC52-07NA2","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G5816449448","display_name":null,"funder_award_id":"DE-AC52-07NA27344","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G5964142490","display_name":null,"funder_award_id":"E-AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G6020043373","display_name":null,"funder_award_id":"DE-AC52-07NA2734","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"},{"id":"https://openalex.org/G8374966015","display_name":null,"funder_award_id":"AC52-07NA27344","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G8658770765","display_name":null,"funder_award_id":"DE-AC52-07NA27","funder_id":"https://openalex.org/F4320306084","funder_display_name":"U.S. Department of Energy"},{"id":"https://openalex.org/G991281326","display_name":null,"funder_award_id":"AC52-07NA27344","funder_id":"https://openalex.org/F4320338286","funder_display_name":"Lawrence Livermore National Laboratory"}],"funders":[{"id":"https://openalex.org/F4320306084","display_name":"U.S. Department of Energy","ror":"https://ror.org/01bj3aw27"},{"id":"https://openalex.org/F4320338286","display_name":"Lawrence Livermore National Laboratory","ror":"https://ror.org/041nk4h53"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4388561281.pdf","grobid_xml":"https://content.openalex.org/works/W4388561281.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W2033787161","https://openalex.org/W2108177987","https://openalex.org/W2119192239","https://openalex.org/W2913114028","https://openalex.org/W2949001633","https://openalex.org/W3123451385","https://openalex.org/W3166421146","https://openalex.org/W4385623175","https://openalex.org/W4388561277","https://openalex.org/W6969482465"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2011676020","https://openalex.org/W2329895846"],"abstract_inverted_index":{"Benchmarking":[0],"is":[1,26],"integral":[2],"to":[3,12,37,57,75,81,99],"procurement":[4],"of":[5,18,42,114,147],"HPC":[6,9,13,21,24,47,66,71,83,104,142,158],"systems,":[7],"communicating":[8],"center":[10],"workloads":[11],"vendors,":[14,106],"and":[15,28,39,62,94,107,118,150],"verifying":[16],"performance":[17],"the":[19,43,82,87,101,111,138,157],"delivered":[20],"systems.":[22,48],"Currently,":[23],"benchmarking":[25,56,93,124,134],"manual":[27],"challenging":[29],"at":[30],"every":[31],"step,":[32],"posing":[33],"a":[34,96,152],"high":[35],"barrier":[36],"entry,":[38],"hampering":[40],"reproducibility":[41],"benchmarks":[44],"across":[45],"different":[46],"In":[49],"this":[50],"paper,":[51],"we":[52],"propose":[53],"collaborative":[54,91,115,132],"continuous":[55,92,116,123,133],"enable":[58],"functional":[59],"reproducibility,":[60],"automation,":[61],"community":[63,128],"collaboration":[64,155],"in":[65,70,141],"benchmarking.":[67],"Recent":[68],"progress":[69],"automation":[72],"allows":[73],"us":[74],"consider":[76],"previously":[77],"unimaginable":[78],"large-scale":[79],"improvements":[80],"ecosystem.":[84],"We":[85,109,130],"define":[86],"minimal":[88],"requirements":[89],"for":[90,127],"develop":[95],"common":[97],"language":[98],"streamline":[100],"interactions":[102],"between":[103],"centers,":[105],"researchers.":[108],"demonstrate":[110],"initial":[112],"implementation":[113],"benchmarking,":[117,143],"introduce":[119],"an":[120],"open":[121],"source":[122],"repository,":[125],"Benchpark,":[126],"collaboration.":[129],"believe":[131],"will":[135],"help":[136],"overcome":[137],"human":[139],"bottleneck":[140],"enabling":[144,151],"better":[145],"evaluation":[146],"our":[148],"systems":[149],"more":[153],"productive":[154],"within":[156],"community.":[159]},"counts_by_year":[{"year":2025,"cited_by_count":8},{"year":2024,"cited_by_count":5}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}