{"id":"https://openalex.org/W2314046539","doi":"https://doi.org/10.1080/10556788.2020.1808977","title":"COCO: a platform for comparing continuous optimizers in a black-box setting","display_name":"COCO: a platform for comparing continuous optimizers in a black-box setting","publication_year":2020,"publication_date":"2020-08-25","ids":{"openalex":"https://openalex.org/W2314046539","doi":"https://doi.org/10.1080/10556788.2020.1808977","mag":"2314046539"},"language":"en","primary_location":{"id":"doi:10.1080/10556788.2020.1808977","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10556788.2020.1808977","pdf_url":null,"source":{"id":"https://openalex.org/S103047102","display_name":"Optimization methods & software","issn_l":"1026-7670","issn":["1026-7670","1029-4937","1055-6788"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Optimization Methods and Software","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1603.08785","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Nikolaus Hansen","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210107641","display_name":"Centre de Math\u00e9matiques Appliqu\u00e9es de l'\u00c9cole polytechnique","ror":"https://ror.org/012e1xn46","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I142476485","https://openalex.org/I4210107641","https://openalex.org/I4210141950","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Nikolaus Hansen","raw_affiliation_strings":["CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","Randopt Team, Inria, Palaiseau, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I142476485","https://openalex.org/I4210107641"]},{"raw_affiliation_string":"Randopt Team, Inria, Palaiseau, France","institution_ids":["https://openalex.org/I1326498283"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Anne Auger","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210107641","display_name":"Centre de Math\u00e9matiques Appliqu\u00e9es de l'\u00c9cole polytechnique","ror":"https://ror.org/012e1xn46","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I142476485","https://openalex.org/I4210107641","https://openalex.org/I4210141950","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Anne Auger","raw_affiliation_strings":["CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","Randopt Team, Inria, Palaiseau, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I142476485","https://openalex.org/I4210107641"]},{"raw_affiliation_string":"Randopt Team, Inria, Palaiseau, France","institution_ids":["https://openalex.org/I1326498283"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Raymond Ros","orcid":null},"institutions":[{"id":"https://openalex.org/I102197404","display_name":"Universit\u00e9 Paris-Sud","ror":"https://ror.org/028rypz17","country_code":"FR","type":"education","lineage":["https://openalex.org/I102197404"]},{"id":"https://openalex.org/I4210144804","display_name":"Laboratoire de Recherche en Informatique","ror":"https://ror.org/04e3ktk27","country_code":"FR","type":"facility","lineage":["https://openalex.org/I102197404","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I4210144804","https://openalex.org/I4210159245"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Raymond Ros","raw_affiliation_strings":["LRI, Universit\u00e9\u00a0Paris-Sud, Orsay, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"LRI, Universit\u00e9\u00a0Paris-Sud, Orsay, France","institution_ids":["https://openalex.org/I102197404","https://openalex.org/I4210144804"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Olaf Mersmann","orcid":null},"institutions":[{"id":"https://openalex.org/I200332995","display_name":"TU Dortmund University","ror":"https://ror.org/01k97gp34","country_code":"DE","type":"education","lineage":["https://openalex.org/I200332995"]}],"countries":["DE"],"is_corresponding":false,"raw_author_name":"Olaf Mersmann","raw_affiliation_strings":["Computational Statistics, TU Dortmund University, Dortmund, Germany"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Computational Statistics, TU Dortmund University, Dortmund, Germany","institution_ids":["https://openalex.org/I200332995"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Tea Tu\u0161ar","orcid":null},"institutions":[{"id":"https://openalex.org/I3006985408","display_name":"Jo\u017eef Stefan Institute","ror":"https://ror.org/05060sz93","country_code":"SI","type":"facility","lineage":["https://openalex.org/I3006985408"]}],"countries":["SI"],"is_corresponding":false,"raw_author_name":"Tea Tu\u0161ar","raw_affiliation_strings":["Jo\u017eef Stefan Institute, Ljubljana, Slovenia"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Jo\u017eef Stefan Institute, Ljubljana, Slovenia","institution_ids":["https://openalex.org/I3006985408"]}]},{"author_position":"last","author":{"id":null,"display_name":"Dimo Brockhoff","orcid":null},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I142476485","display_name":"\u00c9cole Polytechnique","ror":"https://ror.org/05hy3tk52","country_code":"FR","type":"education","lineage":["https://openalex.org/I142476485","https://openalex.org/I4210145102"]},{"id":"https://openalex.org/I4210107641","display_name":"Centre de Math\u00e9matiques Appliqu\u00e9es de l'\u00c9cole polytechnique","ror":"https://ror.org/012e1xn46","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I142476485","https://openalex.org/I4210107641","https://openalex.org/I4210141950","https://openalex.org/I4210145102"]}],"countries":["FR"],"is_corresponding":true,"raw_author_name":"Dimo Brockhoff","raw_affiliation_strings":["CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","Randopt Team, Inria, Palaiseau, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CMAP, CNRS, Ecole Polytechnique, Institut Polytechnique de Paris, Palaiseau, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I142476485","https://openalex.org/I4210107641"]},{"raw_affiliation_string":"Randopt Team, Inria, Palaiseau, France","institution_ids":["https://openalex.org/I1326498283"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I142476485","https://openalex.org/I4210107641"],"apc_list":null,"apc_paid":null,"fwci":20.8917,"has_fulltext":false,"cited_by_count":366,"citation_normalized_percentile":{"value":0.99556973,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"36","issue":"1","first_page":"114","last_page":"144"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10963","display_name":"Advanced Optimization Algorithms Research","score":0.9970999956130981,"subfield":{"id":"https://openalex.org/subfields/2612","display_name":"Numerical Analysis"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10100","display_name":"Metaheuristic Optimization Algorithms Research","score":0.9954000115394592,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.7188000082969666},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7105000019073486},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6139000058174133},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.5717999935150146},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5482000112533569},{"id":"https://openalex.org/keywords/proposition","display_name":"Proposition","score":0.36970001459121704}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.7188000082969666},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7105000019073486},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6139000058174133},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5717999935150146},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5482000112533569},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4747999906539917},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.44589999318122864},{"id":"https://openalex.org/C2777152325","wikidata":"https://www.wikidata.org/wiki/Q108163","display_name":"Proposition","level":2,"score":0.36970001459121704},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.36039999127388},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.3553999960422516},{"id":"https://openalex.org/C2776542497","wikidata":"https://www.wikidata.org/wiki/Q5266672","display_name":"Development (topology)","level":2,"score":0.33709999918937683},{"id":"https://openalex.org/C21780288","wikidata":"https://www.wikidata.org/wiki/Q5139731","display_name":"Coco","level":2,"score":0.32820001244544983},{"id":"https://openalex.org/C68781425","wikidata":"https://www.wikidata.org/wiki/Q2052203","display_name":"Multi-objective optimization","level":2,"score":0.3188000023365021},{"id":"https://openalex.org/C194387892","wikidata":"https://www.wikidata.org/wiki/Q1747770","display_name":"Stochastic optimization","level":2,"score":0.301800012588501},{"id":"https://openalex.org/C137631369","wikidata":"https://www.wikidata.org/wiki/Q7617831","display_name":"Stochastic programming","level":2,"score":0.2962000072002411},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.2816999852657318}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1080/10556788.2020.1808977","is_oa":false,"landing_page_url":"https://doi.org/10.1080/10556788.2020.1808977","pdf_url":null,"source":{"id":"https://openalex.org/S103047102","display_name":"Optimization methods & software","issn_l":"1026-7670","issn":["1026-7670","1029-4937","1055-6788"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320547","host_organization_name":"Taylor & Francis","host_organization_lineage":["https://openalex.org/P4310320547"],"host_organization_lineage_names":["Taylor & Francis"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Optimization Methods and Software","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1603.08785","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1603.08785","pdf_url":"https://arxiv.org/pdf/1603.08785","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:HAL:hal-01294124v4","is_oa":false,"landing_page_url":"https://inria.hal.science/hal-01294124","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL (Le Centre pour la Communication Scientifique Directe)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1294671590","host_organization_name":"Centre National de la Recherche Scientifique","host_organization_lineage":["https://openalex.org/I1294671590"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Optimization Methods and Software, 2021, 36 (1), pp.114-144. ⟨10.1080/10556788.2020.1808977⟩","raw_type":"info:eu-repo/semantics/article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1603.08785","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1603.08785","pdf_url":"https://arxiv.org/pdf/1603.08785","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2249808486","display_name":null,"funder_award_id":"ANR-11-LABX-0056-LMH","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G4895254678","display_name":"LabEx Math\u00e9matique Hadamard","funder_award_id":"ANR-11-LABX-0056","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G5769677123","display_name":"Artificial Intelligence and Intelligent Systems","funder_award_id":"P2-0209","funder_id":"https://openalex.org/F4320322554","funder_display_name":"Javna Agencija za Raziskovalno Dejavnost RS"},{"id":"https://openalex.org/G5933667581","display_name":null,"funder_award_id":"692286","funder_id":"https://openalex.org/F4320332999","funder_display_name":"Horizon 2020 Framework Programme"},{"id":"https://openalex.org/G8780907258","display_name":"Analysis, Improvement and Evaluation of Numerical Black-Box Optimizers","funder_award_id":"ANR-12-MONU-0009","funder_id":"https://openalex.org/F4320320883","funder_display_name":"Agence Nationale de la Recherche"},{"id":"https://openalex.org/G997306319","display_name":"Incorporating real-world problems into the benchmarking of multiobjective optimizers","funder_award_id":"Z2-8177","funder_id":"https://openalex.org/F4320322554","funder_display_name":"Javna Agencija za Raziskovalno Dejavnost RS"}],"funders":[{"id":"https://openalex.org/F4320320883","display_name":"Agence Nationale de la Recherche","ror":"https://ror.org/00rbzpz17"},{"id":"https://openalex.org/F4320322554","display_name":"Javna Agencija za Raziskovalno Dejavnost RS","ror":"https://ror.org/059bp8k51"},{"id":"https://openalex.org/F4320332999","display_name":"Horizon 2020 Framework Programme","ror":"https://ror.org/00k4n6c32"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W1494807122","https://openalex.org/W1543889792","https://openalex.org/W1748199557","https://openalex.org/W1933569933","https://openalex.org/W1995945562","https://openalex.org/W2000552702","https://openalex.org/W2000825106","https://openalex.org/W2005136695","https://openalex.org/W2011301426","https://openalex.org/W2025280025","https://openalex.org/W2027923483","https://openalex.org/W2038210983","https://openalex.org/W2038420231","https://openalex.org/W2056603712","https://openalex.org/W2057443868","https://openalex.org/W2057720927","https://openalex.org/W2057830724","https://openalex.org/W2058922597","https://openalex.org/W2068697786","https://openalex.org/W2078409719","https://openalex.org/W2086252459","https://openalex.org/W2092993939","https://openalex.org/W2094453630","https://openalex.org/W2095478116","https://openalex.org/W2099087967","https://openalex.org/W2103111465","https://openalex.org/W2107230221","https://openalex.org/W2112036188","https://openalex.org/W2112449769","https://openalex.org/W2126105956","https://openalex.org/W2130788193","https://openalex.org/W2140886193","https://openalex.org/W2149454052","https://openalex.org/W2168232938","https://openalex.org/W2171074980","https://openalex.org/W2208154600","https://openalex.org/W2491561177","https://openalex.org/W2496034308","https://openalex.org/W2498232956","https://openalex.org/W2503956893","https://openalex.org/W2754127623","https://openalex.org/W2764251381","https://openalex.org/W2775085189","https://openalex.org/W2825816018","https://openalex.org/W2830370315","https://openalex.org/W2888451463","https://openalex.org/W2901838001","https://openalex.org/W2952437250","https://openalex.org/W2955760803","https://openalex.org/W4211109285","https://openalex.org/W4230471307","https://openalex.org/W4240022917","https://openalex.org/W6634625578"],"related_works":["https://openalex.org/W2780875943","https://openalex.org/W2894431078","https://openalex.org/W4230855886","https://openalex.org/W2281426957","https://openalex.org/W2727407240","https://openalex.org/W2015682171","https://openalex.org/W3095830456","https://openalex.org/W4287626382","https://openalex.org/W3151336090","https://openalex.org/W2922398106"],"abstract_inverted_index":{"We":[0,56,76],"introduce":[1],"COCO,":[2],"an":[3],"open-source":[4],"platform":[5,34,66],"for":[6,50,71],"Comparing":[7],"Continuous":[8],"Optimizers":[9],"in":[10,42],"a":[11,68,88,91,121],"black-box":[12],"setting.":[13],"COCO":[14,82],"aims":[15],"at":[16],"automatizing":[17],"the":[18,29,36,43,58,61,65,85,94,99,108,114,125,130],"tedious":[19],"and":[20,35,47,53,104,129],"repetitive":[21],"task":[22],"of":[23,64,81,87,97,101,110,124],"benchmarking":[24],"numerical":[25],"optimization":[26],"algorithms":[27],"to":[28,40],"greatest":[30],"possible":[31],"extent.":[32],"The":[33],"underlying":[37,78,95],"methodology":[38],"allow":[39],"benchmark":[41],"same":[44],"framework":[45],"deterministic":[46],"stochastic":[48],"solvers":[49],"both":[51],"single":[52],"multiobjective":[54],"optimization.":[55],"present":[57],"rationals":[59],"behind":[60],"(decade-long)":[62],"development":[63],"as":[67,84,90,113],"general":[69],"proposition":[70],"guidelines":[72],"towards":[73],"better":[74],"benchmarking.":[75],"detail":[77],"fundamental":[79],"concepts":[80],"such":[83],"definition":[86],"problem":[89],"function":[92,111],"instance,":[93],"idea":[96],"instances,":[98],"use":[100],"target":[102],"values,":[103],"runtime":[105],"defined":[106],"by":[107],"number":[109],"calls":[112],"central":[115],"performance":[116],"measure.":[117],"Finally,":[118],"we":[119],"give":[120],"quick":[122],"overview":[123],"basic":[126],"code":[127],"structure":[128],"currently":[131],"available":[132],"test":[133],"suites.":[134]},"counts_by_year":[{"year":2026,"cited_by_count":40},{"year":2025,"cited_by_count":87},{"year":2024,"cited_by_count":70},{"year":2023,"cited_by_count":77},{"year":2022,"cited_by_count":61},{"year":2021,"cited_by_count":25},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1}],"updated_date":"2026-07-22T07:51:19.307946","created_date":"2016-06-24T00:00:00"}