{"id":"https://openalex.org/W4225385393","doi":"https://doi.org/10.1145/3520304.3533995","title":"Heed the noise in performance evaluations in neural architecture search","display_name":"Heed the noise in performance evaluations in neural architecture search","publication_year":2022,"publication_date":"2022-07-09","ids":{"openalex":"https://openalex.org/W4225385393","doi":"https://doi.org/10.1145/3520304.3533995"},"language":"en","primary_location":{"id":"doi:10.1145/3520304.3533995","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3520304.3533995","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3520304.3533995","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://dl.acm.org/doi/pdf/10.1145/3520304.3533995","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5006864822","display_name":"Arkadiy Dushatskiy","orcid":"https://orcid.org/0000-0003-0945-0262"},"institutions":[{"id":"https://openalex.org/I1341640284","display_name":"Centrum Wiskunde & Informatica","ror":"https://ror.org/00x7ekv49","country_code":"NL","type":"facility","lineage":["https://openalex.org/I1341640284","https://openalex.org/I2800991832","https://openalex.org/I4405262988"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Arkadiy Dushatskiy","raw_affiliation_strings":["Centrum Wiskunde & Informatica, Amsterdam, the Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Centrum Wiskunde & Informatica, Amsterdam, the Netherlands","institution_ids":["https://openalex.org/I1341640284"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025122230","display_name":"Tanja Alderliesten","orcid":"https://orcid.org/0000-0003-4261-7511"},"institutions":[{"id":"https://openalex.org/I2800006345","display_name":"Leiden University Medical Center","ror":"https://ror.org/05xvt9f17","country_code":"NL","type":"funder","lineage":["https://openalex.org/I2800006345"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Tanja Alderliesten","raw_affiliation_strings":["Leiden University Medical Center, Leiden, the Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Leiden University Medical Center, Leiden, the Netherlands","institution_ids":["https://openalex.org/I2800006345"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5031539135","display_name":"Peter A. N. Bosman","orcid":"https://orcid.org/0000-0002-4186-6666"},"institutions":[{"id":"https://openalex.org/I98358874","display_name":"Delft University of Technology","ror":"https://ror.org/02e2c7k09","country_code":"NL","type":"education","lineage":["https://openalex.org/I98358874"]}],"countries":["NL"],"is_corresponding":false,"raw_author_name":"Peter A. N. Bosman","raw_affiliation_strings":["Delft University of Technology, Delft, The Netherlands"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Delft University of Technology, Delft, The Netherlands","institution_ids":["https://openalex.org/I98358874"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.506,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.6145789,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":null,"issue":null,"first_page":"2104","last_page":"2112"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.9979000091552734,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8211120963096619},{"id":"https://openalex.org/keywords/noise","display_name":"Noise (video)","score":0.7346056699752808},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.7085925340652466},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.637029230594635},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5590006709098816},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5307443141937256},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5222338438034058},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.5122258067131042},{"id":"https://openalex.org/keywords/network-architecture","display_name":"Network architecture","score":0.45883244276046753},{"id":"https://openalex.org/keywords/noise-reduction","display_name":"Noise reduction","score":0.43072041869163513},{"id":"https://openalex.org/keywords/segmentation","display_name":"Segmentation","score":0.4253779351711273},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3834158182144165},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.14502298831939697},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.07849720120429993}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8211120963096619},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.7346056699752808},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.7085925340652466},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.637029230594635},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5590006709098816},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5307443141937256},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5222338438034058},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.5122258067131042},{"id":"https://openalex.org/C193415008","wikidata":"https://www.wikidata.org/wiki/Q639681","display_name":"Network architecture","level":2,"score":0.45883244276046753},{"id":"https://openalex.org/C163294075","wikidata":"https://www.wikidata.org/wiki/Q581861","display_name":"Noise reduction","level":2,"score":0.43072041869163513},{"id":"https://openalex.org/C89600930","wikidata":"https://www.wikidata.org/wiki/Q1423946","display_name":"Segmentation","level":2,"score":0.4253779351711273},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3834158182144165},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.14502298831939697},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.07849720120429993},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3520304.3533995","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3520304.3533995","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3520304.3533995","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2202.02078","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2202.02078","pdf_url":"https://arxiv.org/pdf/2202.02078","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:cwi.nl:32042","is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/32042","pdf_url":"https://ir.cwi.nl/pub/32042/32042.pdf","source":{"id":"https://openalex.org/S7407055335","display_name":"Centrum Wiskunde & Informatica (CWI), the national research institute for mathematics and computer science in the Netherlands","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:tudelft.nl:uuid:81ad6bfc-e91b-4604-942b-bf6acc04b520","is_oa":false,"landing_page_url":"http://resolver.tudelft.nl/uuid:81ad6bfc-e91b-4604-942b-bf6acc04b520","pdf_url":null,"source":{"id":"https://openalex.org/S4306400906","display_name":"Research Repository (Delft University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I98358874","host_organization_name":"Delft University of Technology","host_organization_lineage":["https://openalex.org/I98358874"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"conference paper"}],"best_oa_location":{"id":"doi:10.1145/3520304.3533995","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3520304.3533995","pdf_url":"https://dl.acm.org/doi/pdf/10.1145/3520304.3533995","source":{"id":"https://openalex.org/S4363608771","display_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Genetic and Evolutionary Computation Conference Companion","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5199999809265137,"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure"}],"awards":[{"id":"https://openalex.org/G6216667842","display_name":"FEDMix: Fusible Evolutionary Deep Neural Network Mixture Learning from Distributed Data for Robust Medical Image Analysis","funder_award_id":"628.011.012","funder_id":"https://openalex.org/F4320321800","funder_display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek"}],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W4225385393.pdf"},"referenced_works_count":28,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W1901129140","https://openalex.org/W2113207845","https://openalex.org/W2194775991","https://openalex.org/W2531409750","https://openalex.org/W2804047627","https://openalex.org/W2927980542","https://openalex.org/W2940562175","https://openalex.org/W2951245151","https://openalex.org/W2955425717","https://openalex.org/W2963136578","https://openalex.org/W2965658867","https://openalex.org/W2994731810","https://openalex.org/W3021621050","https://openalex.org/W3035470482","https://openalex.org/W3092618738","https://openalex.org/W3093858143","https://openalex.org/W3112701542","https://openalex.org/W3150058922","https://openalex.org/W3172681723","https://openalex.org/W3177258403","https://openalex.org/W3180629942","https://openalex.org/W3201120310","https://openalex.org/W4214680115","https://openalex.org/W4286914341","https://openalex.org/W4287242089","https://openalex.org/W4287907702","https://openalex.org/W4301762632"],"related_works":["https://openalex.org/W3204184292","https://openalex.org/W3176564347","https://openalex.org/W1985458517","https://openalex.org/W2355833770","https://openalex.org/W3031039437","https://openalex.org/W183202219","https://openalex.org/W3095877357","https://openalex.org/W2072565696","https://openalex.org/W2050451745","https://openalex.org/W2378903222"],"abstract_inverted_index":{"Neural":[0],"Architecture":[1],"Search":[2],"(NAS)":[3],"has":[4],"recently":[5],"become":[6],"a":[7,15,45,54,64,69,102],"topic":[8],"of":[9,44,106,127],"great":[10],"interest.":[11],"However,":[12],"there":[13],"is":[14,50,58,71,178],"potentially":[16],"impactful":[17],"issue":[18],"within":[19],"NAS":[20,107],"that":[21,184],"remains":[22],"largely":[23],"unrecognized:":[24],"noise.":[25],"Due":[26],"to":[27,76],"stochastic":[28],"factors":[29],"in":[30,108,125,187],"neural":[31,46],"network":[32,47,89,128],"initialization,":[33],"training,":[34],"and":[35,96,174],"the":[36,41,117,164],"chosen":[37],"train/validation":[38],"dataset":[39,70],"split,":[40],"performance":[42,86],"evaluation":[43],"architecture,":[48],"which":[49,109,151],"often":[51,172,179],"based":[52,83],"on":[53,84],"single":[55],"learning":[56],"run,":[57],"also":[59],"stochastic.":[60],"This":[61],"may":[62],"have":[63],"particularly":[65],"large":[66],"impact":[67],"if":[68],"small.":[72],"We":[73,98,115,157],"therefore":[74],"propose":[75],"reduce":[77],"this":[78],"noise":[79,112,123,186],"by":[80,194],"evaluating":[81],"architectures":[82,193],"average":[85],"over":[87,139],"multiple":[88],"training":[90,129,141],"runs":[91],"using":[92],"different":[93],"random":[94],"seeds":[95],"cross-validation.":[97],"perform":[99,153],"experiments":[100],"for":[101,121,155],"combinatorial":[103],"optimization":[104],"formulation":[105],"we":[110,132],"vary":[111],"reduction":[113],"levels.":[114],"use":[116,158],"same":[118],"computational":[119],"budget":[120],"each":[122],"level":[124],"terms":[126],"runs,":[130],"i.e.,":[131],"allow":[133],"less":[134],"architecture":[135,188],"evaluations":[136,189],"when":[137],"averaging":[138],"more":[140],"runs.":[142],"Multiple":[143],"search":[144,197],"algorithms":[145,150],"are":[146,171],"considered,":[147],"including":[148],"evolutionary":[149],"generally":[152],"well":[154],"NAS.":[156],"two":[159],"publicly":[160],"available":[161],"datasets":[162,170],"from":[163],"medical":[165],"image":[166],"segmentation":[167],"domain":[168],"where":[169],"limited":[173],"variability":[175],"among":[176],"samples":[177],"high.":[180],"Our":[181],"results":[182],"show":[183],"reducing":[185],"enables":[190],"finding":[191],"better":[192],"all":[195],"considered":[196],"algorithms.":[198]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2023,"cited_by_count":2}],"updated_date":"2026-08-01T09:00:35.917206","created_date":"2025-10-10T00:00:00"}