{"id":"https://openalex.org/W4393769905","doi":"https://doi.org/10.5281/zenodo.10376912","title":"MA-BBOB - Reproducibility and Additional Data","display_name":"MA-BBOB - Reproducibility and Additional Data","publication_year":2023,"publication_date":"2023-12-14","ids":{"openalex":"https://openalex.org/W4393769905","doi":"https://doi.org/10.5281/zenodo.10376912"},"language":"en","primary_location":{"id":"doi:10.5281/zenodo.10376912","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10376912","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"dataset"},"type":"dataset","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.5281/zenodo.10376912","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5075992713","display_name":"Diederick Vermetten","orcid":"https://orcid.org/0000-0003-3040-7162"},"institutions":[{"id":"https://openalex.org/I121797337","display_name":"Leiden University","ror":"https://ror.org/027bh9e22","country_code":"NL","type":"education","lineage":["https://openalex.org/I121797337"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Vermetten, Diederick","raw_affiliation_strings":["Leiden University"],"raw_orcid":"https://orcid.org/0000-0003-3040-7162","affiliations":[{"raw_affiliation_string":"Leiden University","institution_ids":["https://openalex.org/I121797337"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043745558","display_name":"Furong Ye","orcid":"https://orcid.org/0000-0002-8707-4189"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ye, Furong","raw_affiliation_strings":["Chinese Academy of Sciences"],"raw_orcid":"https://orcid.org/0000-0002-8707-4189","affiliations":[{"raw_affiliation_string":"Chinese Academy of Sciences","institution_ids":["https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062646838","display_name":"Thomas B\u00e4ck","orcid":"https://orcid.org/0000-0001-6768-1478"},"institutions":[{"id":"https://openalex.org/I121797337","display_name":"Leiden University","ror":"https://ror.org/027bh9e22","country_code":"NL","type":"education","lineage":["https://openalex.org/I121797337"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"B\u00e4ck, Thomas","raw_affiliation_strings":["Leiden University"],"raw_orcid":"https://orcid.org/0000-0001-6768-1478","affiliations":[{"raw_affiliation_string":"Leiden University","institution_ids":["https://openalex.org/I121797337"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5040561209","display_name":"Carola Doerr","orcid":"https://orcid.org/0000-0002-4981-3227"},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I39804081","display_name":"Sorbonne Universit\u00e9","ror":"https://ror.org/02en5vm52","country_code":"FR","type":"education","lineage":["https://openalex.org/I39804081"]},{"id":"https://openalex.org/I4210159731","display_name":"LIP6","ror":"https://ror.org/05krcen59","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1294671590","https://openalex.org/I39804081","https://openalex.org/I4210159245","https://openalex.org/I4210159731"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Doerr, Carola","raw_affiliation_strings":["Laboratoire de Recherche en Informatique de Paris 6","French National Centre for Scientific Research"],"raw_orcid":"https://orcid.org/0000-0002-4981-3227","affiliations":[{"raw_affiliation_string":"Laboratoire de Recherche en Informatique de Paris 6","institution_ids":["https://openalex.org/I39804081","https://openalex.org/I4210159731"]},{"raw_affiliation_string":"French National Centre for Scientific Research","institution_ids":["https://openalex.org/I1294671590"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11801","display_name":"Reservoir Engineering and Simulation Methods","score":0.156700000166893,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11801","display_name":"Reservoir Engineering and Simulation Methods","score":0.156700000166893,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10522","display_name":"Medical Imaging Techniques and Applications","score":0.1459999978542328,"subfield":{"id":"https://openalex.org/subfields/2741","display_name":"Radiology, Nuclear Medicine and Imaging"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reproducibility","display_name":"Reproducibility","score":0.7763828039169312},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5509065389633179},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.25830352306365967},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.19825679063796997}],"concepts":[{"id":"https://openalex.org/C9893847","wikidata":"https://www.wikidata.org/wiki/Q1425625","display_name":"Reproducibility","level":2,"score":0.7763828039169312},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5509065389633179},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.25830352306365967},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.19825679063796997}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.5281/zenodo.10376912","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10376912","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"dataset"}],"best_oa_location":{"id":"doi:10.5281/zenodo.10376912","is_oa":true,"landing_page_url":"https://doi.org/10.5281/zenodo.10376912","pdf_url":null,"source":{"id":"https://openalex.org/S4306400562","display_name":"Zenodo (CERN European Organization for Nuclear Research)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I67311998","host_organization_name":"European Organization for Nuclear Research","host_organization_lineage":["https://openalex.org/I67311998"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"dataset"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2413717610","https://openalex.org/W1973270181","https://openalex.org/W2417696084","https://openalex.org/W2368782778","https://openalex.org/W2087830269","https://openalex.org/W3106281778","https://openalex.org/W2003643616","https://openalex.org/W638612878","https://openalex.org/W2140154150"],"abstract_inverted_index":{"#":[0],"Reproducability":[1],"files":[2,525,542,582],"for":[3,14,21,264,415,486,512,548,792],"the":[4,11,15,41,52,74,92,98,101,106,114,140,159,168,171,187,195,213,257,277,287,299,302,311,319,323,330,335,342,353,357,360,379,409,417,423,443,449,461,470,490,493,495,504,528,533,539,546,549,568,574,578,584,606,643,678,683,702,721,726,730,736,740,752,756,766,770,775],"paper:":[5],"MABBOB":[6],"extension":[7],"This":[8,31,259,426,716],"document":[9],"details":[10],"reproduction":[12],"steps":[13],"paper":[16,312],"\"MA-BBOB:":[17],"A":[18],"Problem":[19],"Generator":[20],"Black-Box":[22],"Optimization":[23],"through":[24],"Affine":[25],"Combinations":[26],"and":[27,62,64,105,266,368,435,472,509,576,601,604,649,674,747,759],"Shifts\"":[28],"##":[29,275,407],"Dependencies":[30],"project":[32,421],"relies":[33],"on":[34,79,233,243,563,627,647],"a":[35,130,178,250,297],"few":[36],"different":[37,372],"libraries":[38],"to":[39,69,100,129,212,313,328,341,370,432,457,502,521,676,735,765],"achieve":[40],"presented":[42],"results.":[43],"These":[44,374,541],"are":[45,321,375,466,519,543],"as":[46,81,95,97,382,445,447,484,565,580,705,790],"follows:":[47,383],"###":[48,110,174,205,247,488,555,656,743],"IOHexperimenter":[49],"Part":[50],"of":[51,76,121,133,157,180,186,194,219,256,310,359,365,378,463,517,532,609,642,672,751,774,779],"[IOHprofiler](https://iohprofiler.github.io/)":[53],"environment,":[54],"[IOHexperimenter](https://iohprofiler.github.io/IOHexp/)":[55],"provides":[56],"an":[57,261],"interface":[58,99],"between":[59],"optimization":[60,181,268],"algorithms":[61,197,222],"problems,":[63],"adds":[65],"detailed":[66],"logging":[67,93],"functionality":[68],"this":[70,88,148,420,592,691,697,788],"pipeline.":[71],"We":[72,154,191,271],"use":[73,91,120,139,156,185,193,218,255,272,537,671,778],"python-version":[75],"IOHexperimenter,":[77,285],"available":[78,700],"[pip":[80],"'ioh'](https://pypi.org/project/ioh/)":[82],"(we":[83],"used":[84,327,355,501,505,529,544,738,768],"version":[85,273],"0.3.14).":[86],"From":[87],"package,":[89],"we":[90,118,138,183,216,253,536],"component,":[94],"well":[96,446,485,791],"[BBOB":[102],"problem":[103,108],"suite](https://bee22.com/resources/bbob%20functions.pdf)":[104],"MA-BBOB":[107,278,288,534,602],"generator.":[109],"PFlacco":[111],"To":[112,135,176,317],"analyze":[113],"function's":[115],"low-level":[116],"properties,":[117],"make":[119,155,184,192,217,254,777],"Exploratory":[122],"Landscape":[123,638],"Analysis":[124],"(ELA),":[125],"which":[126,161,438,669,784],"gives":[127],"access":[128,177],"wide":[131],"range":[132],"features.":[134],"calculate":[136],"these,":[137],"python-based":[141],"[pflacco](https://github.com/Reiyan/pflacco)":[142],"library":[143,263],"(version":[144,189,235,245],"1.2.2,":[145],"note":[146],"that":[147,682,755],"requires":[149],"python":[150],"3.8":[151],"or":[152],"higher).":[153],"only":[158],"features":[160],"don't":[162],"require":[163],"sampling":[164],"additional":[165],"points":[166],"from":[167,170,198,294,419,478,695,781],"function,":[169],"'classical_ela_features'":[172],"module.":[173],"Nevergrad":[175,214],"variety":[179],"algorihms,":[182],"[Nevergrad](https://github.com/facebookresearch/nevergrad)":[188],"0.4.3.post8).":[190],"following":[196,550,607],"Nevergrads":[199],"optimizers":[200],"module:":[201],"'DifferentialEvolution',":[202],"'DiagonalCMA',":[203],"'RCobyla'":[204],"Modular":[206],"CMA-ES":[207],"+":[208,553],"DE":[209],"In":[210],"addition":[211],"algorithms,":[215],"two":[220],"modular":[221],"frameworks":[223],"in":[224,284,307,356,442,448,469,545,567,640,701,725,739,769,787],"our":[225],"portfolio.":[226],"The":[227,237,412,452,570,594,693],"first":[228],"is":[229,239,260,282,326,422,428,439,455,500,561,667,699,733,749,785],"[Modular":[230,240],"CMA-ES](https://github.com/IOHprofiler/ModularCMAES),":[231],"'modcma'":[232],"pip":[234,244],"1.0.2).":[236],"second":[238],"DE](https://github.com/Dvermetten/ModDE),":[241],"'modde'":[242],"0.0.1).":[246],"IOHanalzyer":[248],"As":[249,296],"final":[251],"requirement,":[252],"[IOHanalyzer](https://github.com/IOHprofiler/IOHanalyzer).":[258],"R-based":[262],"analyzing":[265],"visualizing":[267],"algorithm":[269],"performance.":[270],"0.1.8.4.":[274],"Acessing":[276],"generator":[279,300],"Since":[280],"it":[281],"integrated":[283],"acessing":[286],"functions":[289,320],"can":[290,717],"be":[291,588,687,718,762],"done":[292],"easily":[293],"python.":[295],"default,":[298],"uses":[301,362,475],"function":[303],"creation":[304],"procedure":[305],"described":[306,566],"section":[308,496],"3":[309,465],"sample":[314],"new":[315],"functions.":[316],"ensure":[318],"reusable,":[322],"'instance":[324],"id'":[325],"seed":[329],"generation":[331],"procedure,":[332],"so":[333],"using":[334],"same":[336,343],"id":[337,381],"multiple":[338],"times":[339,431],"leads":[340],"function.":[344],"'''pythonimport":[345],"iohf":[346],"=":[347,350,385,389,391,394,396,399,402,405],"ioh.problem.ManyAffine(1,":[348],"n_variables":[349,404],"5)'''":[351],"However,":[352],"code":[354],"remainer":[358],"repository":[361,704,789],"specific":[363],"settings":[364,491],"weights,":[366,506],"instances":[367,401,531,603],"optima":[369,510],"compare":[371],"settings.":[373],"specified":[376],"instead":[377],"instance":[380,507],"'''pythonxopt":[384],"np.random.uniform(size=(2),":[386],"low=-5,":[387],"high=5)weights":[388],"np.ones(24)/24iids":[390],"list(np.repeat(1,24))":[392],"f2":[393],"ioh.problem.ManyAffine(xopt":[395],"list(xopt),":[397],"weights":[398],"list(weights),":[400],"iids,":[403],"2)'''":[406],"Reproducing":[408],"papers":[410],"results":[411,418,462,579],"core":[413],"file":[414,595,727],"reproducing":[416],"notebook":[424,427,444,776],"'Visualization.ipynb'.":[425],"interrupted":[429],"at":[430],"run":[433],"scripts":[434,547],"collect":[436],"data,":[437],"explained":[440],"both":[441],"sections":[450],"below.":[451],"data":[453,477,498,658,664,694,780],"collection":[454,665],"related":[456],"Section":[458,464,473],"4,":[459],"since":[460],"entirely":[467],"contained":[468],"notebook,":[471,494],"5":[474],"availalbe":[476],"[this":[479],"Zenodo](https://zenodo.org/records/7826036)":[480],"(data":[481],"included":[482,786],"here":[483],"convenience).":[487],"Determine":[489],"usedWithin":[492],"'Setup":[497],"collection'":[499],"generate":[503],"number":[508],"location":[511],"all":[513,598],"pairwise":[514,530],"experiments.":[515],"Each":[516],"these":[518,628],"stored":[520],"their":[522],"corresponding":[523,757],"csv-file.These":[524],"fully":[526],"specify":[527],"suite":[535],"throughout":[538],"paper.":[540],"parts":[551],"(ELA":[552],"Performace)":[554],"Calculating":[556],"ELA":[557,610],"featuresThe":[558],"ELA-feature":[559],"computation":[560,575],"based":[562],"pflacco":[564],"dependencies.":[569],"script:":[571],"'ela_calculation.py'":[572],"runs":[573],"stores":[577],"csv":[581],"(Note:":[583],"'dirname'":[585],"parameter":[586,685,732],"should":[587,686,761],"changed":[589],"before":[590],"running":[591,690,696],"script).":[593],"loops":[596],"over":[597],"selected":[599,679],"BBOB":[600],"gets":[605],"sets":[608],"features:*":[611],"meta":[612],"data*":[613],"distribution*":[614],"level":[615],"set*":[616],"principal":[617],"component":[618],"analysi*":[619],"linear":[620],"model*":[621],"nbc*":[622],"dispersion*":[623],"information_contentFor":[624],"more":[625],"information":[626],"feature":[629],"sets,":[630],"please":[631],"see":[632],"\"Mersmann":[633],"et":[634],"al.":[635],"(2011),":[636],"\u201cExploratory":[637],"Analysis\u201d,":[639],"Proceedings":[641],"13th":[644],"Annual":[645],"Conference":[646],"Genetic":[648],"Evolutionary":[650],"Computation,":[651],"pp.":[652],"829\u2014836.":[653],"ACM":[654],"(http://dx.doi.org/10.1145/2001576.2001690)\"":[655],"Performance":[657],"####":[659,707],"Collect":[660],"performance":[661,663,709],"dataThe":[662,710],"script":[666,698,712,722],"'collect_performance.py',":[668],"makes":[670],"nevergrad":[673],"iohprofiler":[675],"benchmark":[677],"algorithms.":[680],"Note":[681,754],"'rootname'":[684],"modified":[688],"when":[689],"script.":[692,742],"zenodo":[703],"'data.zip'.":[706],"Process":[708],"previous":[711,741],"generates":[713],"IOHanalyzer-compatible":[714],"data.":[715],"processed":[719],"via":[720],"'auc_calculation.py',":[723],"resulting":[724],"'auc.csv'":[728],"if":[729],"dirname":[731],"set":[734],"one":[737],"VisualizationsAll":[744],"remaining":[745],"analysis":[746],"visualization":[748],"part":[750],"notebook.":[753],"directory":[758],"filenames":[760],"updated":[763],"according":[764],"ones":[767],"respective":[771],"scripts.":[772],"Parts":[773],"other":[782],"repositories,":[783],"convenience.":[793]},"counts_by_year":[],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}