{"id":"https://openalex.org/W2078924106","doi":"https://doi.org/10.1145/2668930.2688819","title":"How to Build a Benchmark","display_name":"How to Build a Benchmark","publication_year":2015,"publication_date":"2015-01-12","ids":{"openalex":"https://openalex.org/W2078924106","doi":"https://doi.org/10.1145/2668930.2688819","mag":"2078924106"},"language":"en","primary_location":{"id":"doi:10.1145/2668930.2688819","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2668930.2688819","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th ACM/SPEC International Conference on Performance Engineering","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5015912332","display_name":"J\u00f3akim von Kistowski","orcid":"https://orcid.org/0000-0002-3046-5373"},"institutions":[{"id":"https://openalex.org/I25974101","display_name":"University of W\u00fcrzburg","ror":"https://ror.org/00fbnyb24","country_code":"DE","type":"education","lineage":["https://openalex.org/I25974101"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"J\u00f3akim v. Kistowski","raw_affiliation_strings":["University of W\u00fcrzburg, W\u00fcrzburg, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of W\u00fcrzburg, W\u00fcrzburg, Germany","institution_ids":["https://openalex.org/I25974101"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5057848801","display_name":"Jeremy A. Arnold","orcid":"https://orcid.org/0000-0001-7752-859X"},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jeremy A. Arnold","raw_affiliation_strings":["IBM Corporation, Armonk, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Corporation, Armonk, NY, USA","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074971415","display_name":"Karl Huppler","orcid":null},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Karl Huppler","raw_affiliation_strings":["USA","IBM (United States), Armonk, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"USA","institution_ids":[]},{"raw_affiliation_string":"IBM (United States), Armonk, United States","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087790422","display_name":"Klaus-Dieter Lange","orcid":"https://orcid.org/0009-0002-8068-5021"},"institutions":[{"id":"https://openalex.org/I1324840837","display_name":"Hewlett-Packard (United States)","ror":"https://ror.org/059rn9488","country_code":"US","type":"company","lineage":["https://openalex.org/I1324840837"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Klaus-Dieter Lange","raw_affiliation_strings":["Hewlett-Packard Company, Palo Alto, CA, USA","[Hewlett-Packard Company, Palo Alto, CA, USA]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hewlett-Packard Company, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1324840837"]},{"raw_affiliation_string":"[Hewlett-Packard Company, Palo Alto, CA, USA]","institution_ids":["https://openalex.org/I1324840837"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5078989059","display_name":"John L. Henning","orcid":"https://orcid.org/0000-0001-5063-4915"},"institutions":[{"id":"https://openalex.org/I1342911587","display_name":"Oracle (United States)","ror":"https://ror.org/006c77m33","country_code":"US","type":"company","lineage":["https://openalex.org/I1342911587"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"John L. Henning","raw_affiliation_strings":["Oracle, Redwood City, CA, USA","[Oracle, Redwood City, CA, USA]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Oracle, Redwood City, CA, USA","institution_ids":["https://openalex.org/I1342911587"]},{"raw_affiliation_string":"[Oracle, Redwood City, CA, USA]","institution_ids":["https://openalex.org/I1342911587"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5012663849","display_name":"Paul Cao","orcid":null},"institutions":[{"id":"https://openalex.org/I1324840837","display_name":"Hewlett-Packard (United States)","ror":"https://ror.org/059rn9488","country_code":"US","type":"company","lineage":["https://openalex.org/I1324840837"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Paul Cao","raw_affiliation_strings":["Hewlett-Packard Company, Palo Alto, CA, USA","[Hewlett-Packard Company, Palo Alto, CA, USA]"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Hewlett-Packard Company, Palo Alto, CA, USA","institution_ids":["https://openalex.org/I1324840837"]},{"raw_affiliation_string":"[Hewlett-Packard Company, Palo Alto, CA, USA]","institution_ids":["https://openalex.org/I1324840837"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":11.3816,"has_fulltext":false,"cited_by_count":89,"citation_normalized_percentile":{"value":0.97985789,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"333","last_page":"336"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T11986","display_name":"Scientific Computing and Data Management","score":0.9879999756813049,"subfield":{"id":"https://openalex.org/subfields/1802","display_name":"Information Systems and Management"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.9717000126838684,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10703","display_name":"Business Process Modeling and Analysis","score":0.933899998664856,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.9642142057418823},{"id":"https://openalex.org/keywords/spec#","display_name":"Spec#","score":0.9023653268814087},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7216742038726807},{"id":"https://openalex.org/keywords/workload","display_name":"Workload","score":0.6014125347137451},{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.5319837331771851},{"id":"https://openalex.org/keywords/identification","display_name":"Identification (biology)","score":0.4557201564311981},{"id":"https://openalex.org/keywords/confidentiality","display_name":"Confidentiality","score":0.4221547245979309},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.33851945400238037},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.1298184096813202},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.12868937849998474},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.1182296872138977}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.9642142057418823},{"id":"https://openalex.org/C2778565505","wikidata":"https://www.wikidata.org/wiki/Q2207566","display_name":"Spec#","level":2,"score":0.9023653268814087},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7216742038726807},{"id":"https://openalex.org/C2778476105","wikidata":"https://www.wikidata.org/wiki/Q628539","display_name":"Workload","level":2,"score":0.6014125347137451},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.5319837331771851},{"id":"https://openalex.org/C116834253","wikidata":"https://www.wikidata.org/wiki/Q2039217","display_name":"Identification (biology)","level":2,"score":0.4557201564311981},{"id":"https://openalex.org/C71745522","wikidata":"https://www.wikidata.org/wiki/Q2476929","display_name":"Confidentiality","level":2,"score":0.4221547245979309},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.33851945400238037},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.1298184096813202},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.12868937849998474},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.1182296872138977},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C59822182","wikidata":"https://www.wikidata.org/wiki/Q441","display_name":"Botany","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/2668930.2688819","is_oa":false,"landing_page_url":"https://doi.org/10.1145/2668930.2688819","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 6th ACM/SPEC International Conference on Performance Engineering","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Partnerships for the goals","score":0.44999998807907104,"id":"https://metadata.un.org/sdg/17"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":11,"referenced_works":["https://openalex.org/W1517058456","https://openalex.org/W1554483490","https://openalex.org/W1670932443","https://openalex.org/W2037556552","https://openalex.org/W2075775922","https://openalex.org/W2146930501","https://openalex.org/W2160401437","https://openalex.org/W2188625319","https://openalex.org/W4231717855","https://openalex.org/W4248771687","https://openalex.org/W4285719527"],"related_works":["https://openalex.org/W2142488884","https://openalex.org/W1838930658","https://openalex.org/W2373662841","https://openalex.org/W3082894236","https://openalex.org/W4252566889","https://openalex.org/W2066278930","https://openalex.org/W2378459399","https://openalex.org/W2098005539","https://openalex.org/W2011488966","https://openalex.org/W2107668421"],"abstract_inverted_index":{"Standardized":[0],"benchmarks":[1,17,125],"have":[2],"become":[3],"widely":[4],"accepted":[5],"tools":[6],"for":[7,33,92,115],"the":[8,41,54,61,74,80,89,99,109],"comparison":[9],"of":[10,14,57,63,76,82],"products":[11],"and":[12,24,43,65,78,101,118],"evaluation":[13],"methodologies.":[15],"These":[16],"are":[18,46,133],"created":[19,97],"by":[20,98],"consortia":[21],"like":[22],"SPEC":[23,64,100],"TPC":[25,66,102],"under":[26],"confidentiality":[27],"agreements":[28],"which":[29],"provide":[30,69],"little":[31],"opportunity":[32],"outside":[34],"observers":[35],"to":[36,112,128],"get":[37],"a":[38,70,83,93],"look":[39],"at":[40],"processes":[42],"concerns":[44,56],"that":[45],"prevalent":[47],"in":[48],"benchmark":[49,58,71,116],"development.":[50],"This":[51],"paper":[52],"introduces":[53],"primary":[55,110],"development":[59],"from":[60],"perspectives":[62],"committees.":[67],"We":[68,86,121],"definition,":[72],"outline":[73],"types":[75],"benchmarks,":[77],"explain":[79],"characteristics":[81,90],"good":[84],"benchmark.":[85],"focus":[87],"on":[88],"important":[91],"standardized":[94,124],"benchmark,":[95],"as":[96,126],"consortia.":[103],"To":[104],"this":[105],"end,":[106],"we":[107],"specify":[108],"criteria":[111,132],"be":[113],"employed":[114],"design":[117],"workload":[119],"selection.":[120],"use":[122],"multiple":[123],"examples":[127],"demonstrate":[129],"how":[130],"these":[131],"ensured.":[134]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":11},{"year":2023,"cited_by_count":12},{"year":2022,"cited_by_count":8},{"year":2021,"cited_by_count":12},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":4},{"year":2017,"cited_by_count":4},{"year":2016,"cited_by_count":5}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}