{"id":"https://openalex.org/W2966001554","doi":"https://doi.org/10.1007/s10732-020-09454-w","title":"Sample size calculations for the experimental comparison of multiple algorithms on multiple problem instances","display_name":"Sample size calculations for the experimental comparison of multiple algorithms on multiple problem instances","publication_year":2020,"publication_date":"2020-08-05","ids":{"openalex":"https://openalex.org/W2966001554","doi":"https://doi.org/10.1007/s10732-020-09454-w","mag":"2966001554"},"language":"en","primary_location":{"id":"doi:10.1007/s10732-020-09454-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10732-020-09454-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10732-020-09454-w.pdf","source":{"id":"https://openalex.org/S111418578","display_name":"Journal of Heuristics","issn_l":"1381-1231","issn":["1381-1231","1572-9397"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Heuristics","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10732-020-09454-w.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000238517","display_name":"Felipe Campelo","orcid":"https://orcid.org/0000-0001-8432-4325"},"institutions":[{"id":"https://openalex.org/I110200422","display_name":"Universidade Federal de Minas Gerais","ror":"https://ror.org/0176yjw32","country_code":"BR","type":"education","lineage":["https://openalex.org/I110200422"]},{"id":"https://openalex.org/I169199633","display_name":"Aston University","ror":"https://ror.org/05j0ve876","country_code":"GB","type":"education","lineage":["https://openalex.org/I169199633"]}],"countries":["BR","GB"],"is_corresponding":false,"raw_author_name":"Felipe Campelo","raw_affiliation_strings":["College of Engineering and Physical Sciences, Aston University, Birmingham, B4 7ET, UK","Department of Electrical Engineering, Universidade Federal de Minas Gerais, Belo Horizonte, Brazil","College of Engineering and Physical Sciences, Aston University, Birmingham, UK"],"raw_orcid":"https://orcid.org/0000-0001-8432-4325","affiliations":[{"raw_affiliation_string":"College of Engineering and Physical Sciences, Aston University, Birmingham, B4 7ET, UK","institution_ids":["https://openalex.org/I169199633"]},{"raw_affiliation_string":"Department of Electrical Engineering, Universidade Federal de Minas Gerais, Belo Horizonte, Brazil","institution_ids":["https://openalex.org/I110200422"]},{"raw_affiliation_string":"College of Engineering and Physical Sciences, Aston University, Birmingham, UK","institution_ids":["https://openalex.org/I169199633"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5074650590","display_name":"Elizabeth F. Wanner","orcid":"https://orcid.org/0000-0001-6450-3043"},"institutions":[{"id":"https://openalex.org/I169199633","display_name":"Aston University","ror":"https://ror.org/05j0ve876","country_code":"GB","type":"education","lineage":["https://openalex.org/I169199633"]},{"id":"https://openalex.org/I4210136482","display_name":"Federal Center for Technological Education of Minas Gerais","ror":"https://ror.org/04ch49185","country_code":"BR","type":"education","lineage":["https://openalex.org/I4210136482"]}],"countries":["BR","GB"],"is_corresponding":false,"raw_author_name":"Elizabeth F. Wanner","raw_affiliation_strings":["College of Engineering and Physical Sciences, Aston University, Birmingham, B4 7ET, UK","Department of Computer Engineering, CEFET-MG, Belo Horizonte, Brazil","College of Engineering and Physical Sciences, Aston University, Birmingham, UK"],"raw_orcid":"https://orcid.org/0000-0001-6450-3043","affiliations":[{"raw_affiliation_string":"College of Engineering and Physical Sciences, Aston University, Birmingham, B4 7ET, UK","institution_ids":["https://openalex.org/I169199633"]},{"raw_affiliation_string":"Department of Computer Engineering, CEFET-MG, Belo Horizonte, Brazil","institution_ids":["https://openalex.org/I4210136482"]},{"raw_affiliation_string":"College of Engineering and Physical Sciences, Aston University, Birmingham, UK","institution_ids":["https://openalex.org/I169199633"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":{"value":2890,"currency":"USD","value_usd":2890},"apc_paid":{"value":2890,"currency":"USD","value_usd":2890},"fwci":0.0,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.02679311,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"26","issue":"6","first_page":"851","last_page":"883"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10100","display_name":"Metaheuristic Optimization Algorithms Research","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11596","display_name":"Constraint Satisfaction and Optimization","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/sample-size-determination","display_name":"Sample size determination","score":0.7636406421661377},{"id":"https://openalex.org/keywords/simulated-annealing","display_name":"Simulated annealing","score":0.6594451665878296},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.6220237612724304},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5906065106391907},{"id":"https://openalex.org/keywords/scheduling","display_name":"Scheduling (production processes)","score":0.4906250536441803},{"id":"https://openalex.org/keywords/statistical-power","display_name":"Statistical power","score":0.45626118779182434},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.43466705083847046},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.43292009830474854},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.35315126180648804},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.32971054315567017},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.24342882633209229}],"concepts":[{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.7636406421661377},{"id":"https://openalex.org/C126980161","wikidata":"https://www.wikidata.org/wiki/Q863783","display_name":"Simulated annealing","level":2,"score":0.6594451665878296},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.6220237612724304},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5906065106391907},{"id":"https://openalex.org/C206729178","wikidata":"https://www.wikidata.org/wiki/Q2271896","display_name":"Scheduling (production processes)","level":2,"score":0.4906250536441803},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.45626118779182434},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.43466705083847046},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.43292009830474854},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.35315126180648804},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.32971054315567017},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.24342882633209229},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0}],"mesh":[],"locations_count":7,"locations":[{"id":"doi:10.1007/s10732-020-09454-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10732-020-09454-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10732-020-09454-w.pdf","source":{"id":"https://openalex.org/S111418578","display_name":"Journal of Heuristics","issn_l":"1381-1231","issn":["1381-1231","1572-9397"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Heuristics","raw_type":"journal-article"},{"id":"pmh:oai:research-information.bris.ac.uk:openaire_cris_publications/00b6250b-8cdf-4e5b-a875-c677ff43e284","is_oa":false,"landing_page_url":"https://research-information.bris.ac.uk/en/publications/00b6250b-8cdf-4e5b-a875-c677ff43e284","pdf_url":null,"source":{"id":"https://openalex.org/S4306400895","display_name":"Bristol Research (University of Bristol)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I36234482","host_organization_name":"University of Bristol","host_organization_lineage":["https://openalex.org/I36234482"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Campelo, F & Wanner, E F 2020, 'Sample size calculations for the experimental comparison of multiple algorithms on multiple problem instances', Journal of Heuristics. https://doi.org/10.1007/s10732-020-09454-w","raw_type":"info:eu-repo/semantics/publishedVersion"},{"id":"pmh:oai:publications.aston.ac.uk:41787","is_oa":true,"landing_page_url":"https://publications.aston.ac.uk/view/author/e731149615ac85d43265dfea83ad1aec.html>","pdf_url":"https://publications.aston.ac.uk/id/eprint/41787/1/Campelo_Wanner2020_Article_SampleSizeCalculationsForTheEx.pdf","source":{"id":"https://openalex.org/S4306400483","display_name":"Aston Publications Explorer (Aston University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I169199633","host_organization_name":"Aston University","host_organization_lineage":["https://openalex.org/I169199633"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"acceptedVersion","is_accepted":true,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"pmh:oai:arXiv.org:1908.01720","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1908.01720","pdf_url":"https://arxiv.org/pdf/1908.01720","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2966001554","is_oa":true,"landing_page_url":"https://arxiv.org/abs/1908.01720","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"pmh:oai:RePEc:spr:joheur:v:26:y:2020:i:6:d:10.1007_s10732-020-09454-w","is_oa":false,"landing_page_url":"http://link.springer.com/10.1007/s10732-020-09454-w","pdf_url":null,"source":{"id":"https://openalex.org/S4306401271","display_name":"RePEc: Research Papers in Economics","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I77793887","host_organization_name":"Federal Reserve Bank of St. Louis","host_organization_lineage":["https://openalex.org/I77793887"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"article"},{"id":"doi:10.48550/arxiv.1908.01720","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1908.01720","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1007/s10732-020-09454-w","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10732-020-09454-w","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10732-020-09454-w.pdf","source":{"id":"https://openalex.org/S111418578","display_name":"Journal of Heuristics","issn_l":"1381-1231","issn":["1381-1231","1572-9397"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Journal of Heuristics","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4775155698","display_name":null,"funder_award_id":"404988/2016-4","funder_id":"https://openalex.org/F4320322025","funder_display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico"},{"id":"https://openalex.org/G7906702410","display_name":null,"funder_award_id":"APQ-01099-16","funder_id":"https://openalex.org/F4320322980","funder_display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais"},{"id":"https://openalex.org/G8842991335","display_name":null,"funder_award_id":"RF-2018-527/9","funder_id":"https://openalex.org/F4320319993","funder_display_name":"Leverhulme Trust"}],"funders":[{"id":"https://openalex.org/F4320319993","display_name":"Leverhulme Trust","ror":"https://ror.org/012mzw131"},{"id":"https://openalex.org/F4320322025","display_name":"Conselho Nacional de Desenvolvimento Cient\u00edfico e Tecnol\u00f3gico","ror":"https://ror.org/03swz6y49"},{"id":"https://openalex.org/F4320322980","display_name":"Funda\u00e7\u00e3o de Amparo \u00e0 Pesquisa do Estado de Minas Gerais","ror":"https://ror.org/00nc55f03"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2966001554.pdf","grobid_xml":"https://content.openalex.org/works/W2966001554.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W170314049","https://openalex.org/W578092267","https://openalex.org/W964861289","https://openalex.org/W1475104872","https://openalex.org/W1488422606","https://openalex.org/W1625914768","https://openalex.org/W1655938888","https://openalex.org/W1678759862","https://openalex.org/W1975325658","https://openalex.org/W1981457167","https://openalex.org/W1981832641","https://openalex.org/W1983615075","https://openalex.org/W2035929796","https://openalex.org/W2049736842","https://openalex.org/W2061554433","https://openalex.org/W2062450536","https://openalex.org/W2078214107","https://openalex.org/W2079028546","https://openalex.org/W2087228724","https://openalex.org/W2092993939","https://openalex.org/W2095478116","https://openalex.org/W2096166399","https://openalex.org/W2099913788","https://openalex.org/W2103162689","https://openalex.org/W2103824021","https://openalex.org/W2120765535","https://openalex.org/W2121044470","https://openalex.org/W2121492055","https://openalex.org/W2135445066","https://openalex.org/W2137098853","https://openalex.org/W2139156166","https://openalex.org/W2142848952","https://openalex.org/W2155640378","https://openalex.org/W2156773695","https://openalex.org/W2165466912","https://openalex.org/W2469183162","https://openalex.org/W2490627639","https://openalex.org/W2527882685","https://openalex.org/W2887566942","https://openalex.org/W2962236927","https://openalex.org/W2996208068","https://openalex.org/W4251464645","https://openalex.org/W6606494994"],"related_works":["https://openalex.org/W2087512756","https://openalex.org/W2059130317","https://openalex.org/W1993469070","https://openalex.org/W2133619163","https://openalex.org/W2608963621","https://openalex.org/W2092974964","https://openalex.org/W2054904540","https://openalex.org/W2102431434","https://openalex.org/W154035418","https://openalex.org/W2322185337","https://openalex.org/W2085093899","https://openalex.org/W2512939052","https://openalex.org/W2107522046","https://openalex.org/W2129309709","https://openalex.org/W1548006809","https://openalex.org/W3023942034","https://openalex.org/W2100655446","https://openalex.org/W2080501133","https://openalex.org/W2026507776","https://openalex.org/W2263635495"],"abstract_inverted_index":{"Abstract":[0],"This":[1,29,82],"work":[2],"presents":[3,85],"a":[4,23,58,111,130,135],"statistically":[5],"principled":[6],"method":[7],"for":[8,88,134,150],"estimating":[9],"the":[10,16,42,68,103,115,124,144,147,155],"required":[11,105],"number":[12,106],"of":[13,19,27,107,117,126,129,137,146,158],"instances":[14],"in":[15,78,114,154],"experimental":[17,156],"comparison":[18,157],"multiple":[20],"algorithms":[21,55],"on":[22,41,92,96],"given":[24],"problem":[25],"class":[26,136],"interest.":[28],"approach":[30,87],"generalises":[31],"earlier":[32],"results":[33],"by":[34],"allowing":[35],"researchers":[36],"to":[37,51,66,110,142],"design":[38],"experiments":[39],"based":[40,95],"desired":[43,74,112],"best,":[44],"worst,":[45],"mean":[46],"or":[47],"median-case":[48],"statistical":[49],"power":[50],"detect":[52],"differences":[53],"between":[54],"larger":[56],"than":[57],"certain":[59],"threshold.":[60],"Holm\u2019s":[61],"step-down":[62],"procedure":[63],"is":[64,140],"used":[65,141],"maintain":[67],"overall":[69],"significance":[70],"level":[71],"controlled":[72],"at":[73],"levels,":[75],"without":[76],"resulting":[77],"overly":[79],"conservative":[80],"experiments.":[81],"paper":[83],"also":[84],"an":[86],"sampling":[89],"each":[90,93],"algorithm":[91],"instance,":[94],"optimal":[97],"sample":[98,151],"size":[99,152],"ratios":[100],"that":[101],"minimise":[102],"total":[104],"runs":[108],"subject":[109],"accuracy":[113],"estimation":[116],"paired":[118],"differences.":[119],"A":[120],"case":[121],"study":[122],"investigating":[123],"effect":[125],"21":[127],"variants":[128],"custom-tailored":[131],"Simulated":[132],"Annealing":[133],"scheduling":[138],"problems":[139],"illustrate":[143],"application":[145],"proposed":[148],"methods":[149],"calculations":[153],"algorithms.":[159]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}