{"id":"https://openalex.org/W4288080207","doi":"https://doi.org/10.1145/3377929.3389974","title":"Towards realistic optimization benchmarks","display_name":"Towards realistic optimization benchmarks","publication_year":2020,"publication_date":"2020-07-08","ids":{"openalex":"https://openalex.org/W4288080207","doi":"https://doi.org/10.1145/3377929.3389974"},"language":"en","primary_location":{"id":"doi:10.1145/3377929.3389974","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3377929.3389974","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},"type":"preprint","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2004.06395","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5011658906","display_name":"Koen van der Blom","orcid":"https://orcid.org/0000-0002-4653-0707"},"institutions":[{"id":"https://openalex.org/I121797337","display_name":"Leiden University","ror":"https://ror.org/027bh9e22","country_code":"NL","type":"education","lineage":["https://openalex.org/I121797337"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Koen van der Blom","raw_affiliation_strings":["Leiden University, Leiden, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leiden University, Leiden, The Netherlands","institution_ids":["https://openalex.org/I121797337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035749164","display_name":"Timo M. Deist","orcid":"https://orcid.org/0000-0003-0057-1535"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Timo M. Deist","raw_affiliation_strings":["Informatica Amsterdam, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Informatica Amsterdam, The Netherlands","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011577527","display_name":"Tea Tu\u0161ar","orcid":"https://orcid.org/0000-0002-6495-006X"},"institutions":[{"id":"https://openalex.org/I3006985408","display_name":"Jo\u017eef Stefan Institute","ror":"https://ror.org/05060sz93","country_code":"SI","type":"facility","lineage":["https://openalex.org/I3006985408"]}],"countries":["SI"],"is_corresponding":false,"raw_author_name":"Tea Tu\u0161ar","raw_affiliation_strings":["Jozef Stefan Institute, Ljubljana, Slovenia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jozef Stefan Institute, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I3006985408"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5047294765","display_name":"Mariapia Marchi","orcid":"https://orcid.org/0000-0002-7107-9073"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mariapia Marchi","raw_affiliation_strings":["ESTECO SpA, Trieste, Italy"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"ESTECO SpA, Trieste, Italy","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004957967","display_name":"Yusuke Nojima","orcid":"https://orcid.org/0000-0003-4853-1305"},"institutions":[{"id":"https://openalex.org/I15807432","display_name":"Osaka Prefecture University","ror":"https://ror.org/02cf1je33","country_code":"JP","type":"education","lineage":["https://openalex.org/I15807432"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Yusuke Nojima","raw_affiliation_strings":["Osaka Prefecture University, Sakai, Osaka, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Osaka Prefecture University, Sakai, Osaka, Japan","institution_ids":["https://openalex.org/I15807432"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016516233","display_name":"Akira Oyama","orcid":"https://orcid.org/0000-0003-1432-8165"},"institutions":[{"id":"https://openalex.org/I2800865746","display_name":"Japan Aerospace Exploration Agency","ror":"https://ror.org/059yhyy33","country_code":"JP","type":"facility","lineage":["https://openalex.org/I2800865746"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Akira Oyama","raw_affiliation_strings":["Japan Aerospace Exploration Agency, Sagamihara, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Japan Aerospace Exploration Agency, Sagamihara, Japan","institution_ids":["https://openalex.org/I2800865746"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041698869","display_name":"Vanessa Volz","orcid":"https://orcid.org/0000-0002-4919-4374"},"institutions":[{"id":"https://openalex.org/I2803050950","display_name":"Danish Ministry of Defence","ror":"https://ror.org/02ncgfj77","country_code":"DK","type":"government","lineage":["https://openalex.org/I2803050950"]}],"countries":["DK"],"is_corresponding":false,"raw_author_name":"Vanessa Volz","raw_affiliation_strings":["modl.ai, Copenhagen, Denmark"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"modl.ai, Copenhagen, Denmark","institution_ids":["https://openalex.org/I2803050950"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070705949","display_name":"Boris Naujoks","orcid":"https://orcid.org/0000-0002-8969-4795"},"institutions":[{"id":"https://openalex.org/I102520234","display_name":"TH K\u00f6ln - University of Applied Sciences","ror":"https://ror.org/014nnvj65","country_code":"DE","type":"education","lineage":["https://openalex.org/I102520234"]},{"id":"https://openalex.org/I4210088388","display_name":"Kreiskrankenhaus Gummersbach","ror":"https://ror.org/007857862","country_code":"DE","type":"company","lineage":["https://openalex.org/I4210088388"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Boris Naujoks","raw_affiliation_strings":["TH K\u00f6ln, Gummersbach, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"TH K\u00f6ln, Gummersbach, Germany","institution_ids":["https://openalex.org/I4210088388","https://openalex.org/I102520234"]}]}],"institutions":[],"countries_distinct_count":5,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5011658906"],"corresponding_institution_ids":["https://openalex.org/I121797337"],"apc_list":null,"apc_paid":null,"fwci":0.7282,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.78436019,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":null,"issue":null,"first_page":"293","last_page":"294"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10100","display_name":"Metaheuristic Optimization Algorithms Research","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9886999726295471,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8883507251739502},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7802727818489075},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5711532831192017},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.5306296348571777},{"id":"https://openalex.org/keywords/point","display_name":"Point (geometry)","score":0.5204184651374817},{"id":"https://openalex.org/keywords/data-science","display_name":"Data science","score":0.4702930450439453},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.4673396050930023},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.37124505639076233},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3229018449783325},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.10617858171463013},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.10085251927375793},{"id":"https://openalex.org/keywords/computer-security","display_name":"Computer security","score":0.08773922920227051}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8883507251739502},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7802727818489075},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5711532831192017},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.5306296348571777},{"id":"https://openalex.org/C28719098","wikidata":"https://www.wikidata.org/wiki/Q44946","display_name":"Point (geometry)","level":2,"score":0.5204184651374817},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.4702930450439453},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.4673396050930023},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.37124505639076233},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3229018449783325},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.10617858171463013},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.10085251927375793},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.08773922920227051},{"id":"https://openalex.org/C78519656","wikidata":"https://www.wikidata.org/wiki/Q101333","display_name":"Mechanical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3377929.3389974","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3377929.3389974","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 2020 Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2004.06395","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.06395","pdf_url":"https://arxiv.org/pdf/2004.06395","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:scholarlypublications.universiteitleiden.nl:item_3307602","is_oa":false,"landing_page_url":"https://hdl.handle.net/1887/3307602","pdf_url":null,"source":{"id":"https://openalex.org/S4306400850","display_name":"Leiden Repository (Leiden University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I121797337","host_organization_name":"Leiden University","host_organization_lineage":["https://openalex.org/I121797337"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 2020 Genetic and Evolutionary Computation Conference Companion","raw_type":"Still image (photo, video, movie)"},{"id":"pmh:ul:oai:scholarlypublications.universiteitleiden.nl:item_3307602","is_oa":false,"landing_page_url":"http://hdl.handle.net/1887/3307602","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the 2020 Genetic and Evolutionary Computation Conference Companion, 293 - -294. New York, NY, USA: ACM","raw_type":"info:eu-repo/semantics/other"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2004.06395","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2004.06395","pdf_url":"https://arxiv.org/pdf/2004.06395","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Partnerships for the goals","id":"https://metadata.un.org/sdg/17","score":0.4300000071525574}],"awards":[{"id":"https://openalex.org/G2312103861","display_name":null,"funder_award_id":"Z2-8177 and BI-DE/20-21-019","funder_id":"https://openalex.org/F4320322554","funder_display_name":"Javna Agencija za Raziskovalno Dejavnost RS"},{"id":"https://openalex.org/G4748425923","display_name":null,"funder_award_id":"57515062","funder_id":"https://openalex.org/F4320334761","funder_display_name":"German Academic Exchange Service London"}],"funders":[{"id":"https://openalex.org/F4320321780","display_name":"Ministerie van Economische Zaken","ror":"https://ror.org/00bc6bw85"},{"id":"https://openalex.org/F4320322554","display_name":"Javna Agencija za Raziskovalno Dejavnost RS","ror":"https://ror.org/059bp8k51"},{"id":"https://openalex.org/F4320334761","display_name":"German Academic Exchange Service London","ror":"https://ror.org/039djdh30"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":2,"referenced_works":["https://openalex.org/W2731819575","https://openalex.org/W2969026648"],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"Benchmarks":[0],"are":[1],"a":[2,36,43,57,71],"useful":[3],"tool":[4],"for":[5,92],"empirical":[6],"performance":[7,34],"comparisons.":[8],"However,":[9],"one":[10],"of":[11,15,53,82,107,110],"the":[12,80,101,129,136],"main":[13],"shortcomings":[14],"existing":[16],"benchmarks":[17,84],"is":[18,95,131],"that":[19,74],"it":[20],"remains":[21],"largely":[22],"unclear":[23],"how":[24],"they":[25],"relate":[26],"to":[27,50,76,96,100,103,133],"real-world":[28,45,54,60],"problems.":[29,66],"What":[30],"does":[31],"an":[32,105],"algorithm's":[33],"on":[35,42,59,68],"benchmark":[37,123],"say":[38],"about":[39],"its":[40],"potential":[41],"specific":[44],"problem?":[46],"This":[47],"work":[48,94],"aims":[49],"identify":[51],"properties":[52],"problems":[55],"through":[56],"questionnaire":[58,102,137],"single-,":[61],"multi-,":[62],"and":[63],"many-objective":[64],"optimization":[65],"Based":[67],"initial":[69],"responses,":[70],"few":[72],"challenges":[73],"have":[75],"be":[77,87,119],"considered":[78],"in":[79,121,135],"design":[81],"realistic":[83],"can":[85,117],"already":[86],"identified.":[88],"A":[89],"key":[90],"point":[91],"future":[93],"gather":[97,126],"more":[98,127],"responses":[99],"allow":[104],"analysis":[106],"common":[108,115],"combinations":[109,116],"properties.":[111],"In":[112],"turn,":[113],"such":[114],"then":[118],"included":[120],"improved":[122],"suites.":[124],"To":[125],"data,":[128],"reader":[130],"invited":[132],"participate":[134],"at:":[138],"https://tinyurl.com/opt-survey":[139]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2026-04-29T09:16:38.111599","created_date":"2022-07-28T00:00:00"}