{"id":"https://openalex.org/W3082154327","doi":"https://doi.org/10.1109/tpami.2021.3054824","title":"NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size","display_name":"NATS-Bench: Benchmarking NAS Algorithms for Architecture Topology and Size","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3082154327","doi":"https://doi.org/10.1109/tpami.2021.3054824","mag":"3082154327","pmid":"https://pubmed.ncbi.nlm.nih.gov/33497330"},"language":"en","primary_location":{"id":"doi:10.1109/tpami.2021.3054824","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2021.3054824","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2009.00437","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5069837273","display_name":"Xuanyi Dong","orcid":"https://orcid.org/0000-0001-9272-1590"},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":true,"raw_author_name":"Xuanyi Dong","raw_affiliation_strings":["Centre for Artificial Intelligence, University of Technology Sydney, 1994 Sydney, New South Wales, Australia, 2007 (e-mail: dongxuanyi888@icloud.com)","School of Computer Science, University of Technology Sydney, NSW, Australia"],"affiliations":[{"raw_affiliation_string":"Centre for Artificial Intelligence, University of Technology Sydney, 1994 Sydney, New South Wales, Australia, 2007 (e-mail: dongxuanyi888@icloud.com)","institution_ids":["https://openalex.org/I114017466"]},{"raw_affiliation_string":"School of Computer Science, University of Technology Sydney, NSW, Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002822427","display_name":"Lu Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Lu Liu","raw_affiliation_strings":["Center for Artificial Intelligence, University of Technology Sydney, 1994 Sydney, New South Wales, Australia, 2007 (e-mail: LU.LIU.CS@ICLOUD.COM)","School of Computer Science, University of Technology Sydney, NSW, Australia"],"affiliations":[{"raw_affiliation_string":"Center for Artificial Intelligence, University of Technology Sydney, 1994 Sydney, New South Wales, Australia, 2007 (e-mail: LU.LIU.CS@ICLOUD.COM)","institution_ids":["https://openalex.org/I114017466"]},{"raw_affiliation_string":"School of Computer Science, University of Technology Sydney, NSW, Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051832847","display_name":"Katarzyna Musia\u0142","orcid":"https://orcid.org/0000-0001-6038-7647"},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Katarzyna Musial","raw_affiliation_strings":["School of Software, University of Technology Sydney Faculty of Engineering and Information Technology, 120558 Sydney, New South Wales, Australia, 2007 (e-mail: katarzyna.musial-gabrys@uts.edu.au)","School of Computer Science, University of Technology Sydney, NSW, Australia"],"affiliations":[{"raw_affiliation_string":"School of Software, University of Technology Sydney Faculty of Engineering and Information Technology, 120558 Sydney, New South Wales, Australia, 2007 (e-mail: katarzyna.musial-gabrys@uts.edu.au)","institution_ids":["https://openalex.org/I114017466"]},{"raw_affiliation_string":"School of Computer Science, University of Technology Sydney, NSW, Australia","institution_ids":["https://openalex.org/I114017466"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5083181106","display_name":"Bogdan Gabry\u015b","orcid":"https://orcid.org/0000-0002-0790-2846"},"institutions":[{"id":"https://openalex.org/I114017466","display_name":"University of Technology Sydney","ror":"https://ror.org/03f0f6041","country_code":"AU","type":"education","lineage":["https://openalex.org/I114017466"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Bogdan Gabrys","raw_affiliation_strings":["Advanced Analytics Institute, University of Technology Sydney Faculty of Engineering and Information Technology, 120558 Sydney, New South Wales, Australia, (e-mail: bogdan.gabrys@uts.edu.au)","School of Computer Science, University of Technology Sydney, NSW, Australia"],"affiliations":[{"raw_affiliation_string":"Advanced Analytics Institute, University of Technology Sydney Faculty of Engineering and Information Technology, 120558 Sydney, New South Wales, Australia, (e-mail: bogdan.gabrys@uts.edu.au)","institution_ids":["https://openalex.org/I114017466"]},{"raw_affiliation_string":"School of Computer Science, University of Technology Sydney, NSW, Australia","institution_ids":["https://openalex.org/I114017466"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5069837273"],"corresponding_institution_ids":["https://openalex.org/I114017466"],"apc_list":null,"apc_paid":null,"fwci":11.6512,"has_fulltext":false,"cited_by_count":158,"citation_normalized_percentile":{"value":0.9906761,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":94,"max":100},"biblio":{"volume":"44","issue":"7","first_page":"1","last_page":"1"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11948","display_name":"Machine Learning in Materials Science","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/2505","display_name":"Materials Chemistry"},"field":{"id":"https://openalex.org/fields/25","display_name":"Materials Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8656641244888306},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7649054527282715},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6846092343330383},{"id":"https://openalex.org/keywords/architecture","display_name":"Architecture","score":0.5981485247612},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.5125858187675476},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.49087291955947876},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4633617401123047},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4536759555339813},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.38781827688217163}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8656641244888306},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7649054527282715},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6846092343330383},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.5981485247612},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.5125858187675476},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.49087291955947876},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4633617401123047},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4536759555339813},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.38781827688217163},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C153349607","wikidata":"https://www.wikidata.org/wiki/Q36649","display_name":"Visual arts","level":1,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D009474","descriptor_name":"Neurons","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009474","descriptor_name":"Neurons","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D009474","descriptor_name":"Neurons","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D010936","descriptor_name":"Plant Extracts","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D010936","descriptor_name":"Plant Extracts","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D010936","descriptor_name":"Plant Extracts","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D016571","descriptor_name":"Neural Networks, Computer","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D019985","descriptor_name":"Benchmarking","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false}],"locations_count":4,"locations":[{"id":"doi:10.1109/tpami.2021.3054824","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tpami.2021.3054824","pdf_url":null,"source":{"id":"https://openalex.org/S199944782","display_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","issn_l":"0162-8828","issn":["0162-8828","1939-3539","2160-9292"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Pattern Analysis and Machine Intelligence","raw_type":"journal-article"},{"id":"pmid:33497330","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/33497330","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on pattern analysis and machine intelligence","raw_type":null},{"id":"pmh:oai:arXiv.org:2009.00437","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2009.00437","pdf_url":"https://arxiv.org/pdf/2009.00437","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:opus.lib.uts.edu.au:10453/147164","is_oa":false,"landing_page_url":"http://hdl.handle.net/10453/147164","pdf_url":null,"source":{"id":"https://openalex.org/S4306401357","display_name":"UTS ePRESS (University of Technology Sydney)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I114017466","host_organization_name":"University of Technology Sydney","host_organization_lineage":["https://openalex.org/I114017466"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"Journal Article"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2009.00437","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2009.00437","pdf_url":"https://arxiv.org/pdf/2009.00437","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"display_name":"Industry, innovation and infrastructure","score":0.5799999833106995,"id":"https://metadata.un.org/sdg/9"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":109,"referenced_works":["https://openalex.org/W1686810756","https://openalex.org/W1836465849","https://openalex.org/W1979354511","https://openalex.org/W2097117768","https://openalex.org/W2097998348","https://openalex.org/W2117539524","https://openalex.org/W2119717200","https://openalex.org/W2163605009","https://openalex.org/W2194775991","https://openalex.org/W2553303224","https://openalex.org/W2565639579","https://openalex.org/W2604998962","https://openalex.org/W2618530766","https://openalex.org/W2741430497","https://openalex.org/W2748513770","https://openalex.org/W2765407302","https://openalex.org/W2773706593","https://openalex.org/W2810075754","https://openalex.org/W2886851211","https://openalex.org/W2890166761","https://openalex.org/W2897818259","https://openalex.org/W2915992092","https://openalex.org/W2916118939","https://openalex.org/W2944786596","https://openalex.org/W2945403477","https://openalex.org/W2949117887","https://openalex.org/W2949941638","https://openalex.org/W2951104886","https://openalex.org/W2951245151","https://openalex.org/W2952634764","https://openalex.org/W2960010704","https://openalex.org/W2962746461","https://openalex.org/W2962835968","https://openalex.org/W2962971773","https://openalex.org/W2963136578","https://openalex.org/W2963137684","https://openalex.org/W2963263347","https://openalex.org/W2963285940","https://openalex.org/W2963374479","https://openalex.org/W2963399829","https://openalex.org/W2963403868","https://openalex.org/W2963423218","https://openalex.org/W2963536136","https://openalex.org/W2963628712","https://openalex.org/W2963815651","https://openalex.org/W2963821229","https://openalex.org/W2963918968","https://openalex.org/W2964081807","https://openalex.org/W2964294659","https://openalex.org/W2964515685","https://openalex.org/W2965658867","https://openalex.org/W2970790493","https://openalex.org/W2972268045","https://openalex.org/W2980270353","https://openalex.org/W2981698279","https://openalex.org/W2987301155","https://openalex.org/W2994749257","https://openalex.org/W2994842046","https://openalex.org/W2995727387","https://openalex.org/W2996481211","https://openalex.org/W2999270366","https://openalex.org/W3033161802","https://openalex.org/W3034609471","https://openalex.org/W3034971973","https://openalex.org/W3035400692","https://openalex.org/W3092207651","https://openalex.org/W3098607469","https://openalex.org/W3118608800","https://openalex.org/W3172317602","https://openalex.org/W4230563027","https://openalex.org/W4241811150","https://openalex.org/W4246193833","https://openalex.org/W4255898812","https://openalex.org/W4287900329","https://openalex.org/W4287907702","https://openalex.org/W4288357194","https://openalex.org/W4289763996","https://openalex.org/W4297778814","https://openalex.org/W4307823382","https://openalex.org/W4385245566","https://openalex.org/W6637373629","https://openalex.org/W6638667902","https://openalex.org/W6674385629","https://openalex.org/W6685444988","https://openalex.org/W6726497184","https://openalex.org/W6729956949","https://openalex.org/W6730169791","https://openalex.org/W6739901393","https://openalex.org/W6741978826","https://openalex.org/W6743495212","https://openalex.org/W6745728296","https://openalex.org/W6751421292","https://openalex.org/W6752515464","https://openalex.org/W6754484989","https://openalex.org/W6755501315","https://openalex.org/W6759402996","https://openalex.org/W6759828284","https://openalex.org/W6760018943","https://openalex.org/W6762264498","https://openalex.org/W6762610423","https://openalex.org/W6766225098","https://openalex.org/W6767064347","https://openalex.org/W6767472356","https://openalex.org/W6769344749","https://openalex.org/W6771859737","https://openalex.org/W6772103215","https://openalex.org/W6783671563","https://openalex.org/W6785401007","https://openalex.org/W6787972765"],"related_works":["https://openalex.org/W4238897586","https://openalex.org/W435179959","https://openalex.org/W2619091065","https://openalex.org/W2059640416","https://openalex.org/W1490753184","https://openalex.org/W2284465472","https://openalex.org/W2291782699","https://openalex.org/W1993948687","https://openalex.org/W2000169967","https://openalex.org/W2112883198"],"abstract_inverted_index":{"Neural":[0],"architecture":[1,32,139,144],"search":[2,81,131,169],"(NAS)":[3],"has":[4,11,54],"attracted":[5],"a":[6,19,99,112,204,218],"lot":[7],"of":[8,22,39,47,57,62,65,91,101,133,153,158,164,176,208],"attention":[9],"and":[10,31,51,83,95,120,141,161,189,221],"been":[12,35],"illustrated":[13],"to":[14,210],"bring":[15],"tangible":[16],"benefits":[17],"in":[18,24,156,167,217],"large":[20],"number":[21],"applications":[23],"the":[25,40,45,52,66,70,88,92,96,102,130,151,168,174,194],"past":[26],"few":[27],"years.":[28],"Architecture":[29],"topology":[30,119,140],"size":[33,145],"have":[34],"regarded":[36],"as":[37],"two":[38],"most":[41],"important":[42],"aspects":[43,64],"for":[44,60,117,122,138,143,197],"performance":[46,71,90,162],"deep":[48],"learning":[49],"models":[50],"community":[53,207],"spawned":[55],"lots":[56],"searching":[58,75,103,116],"algorithms":[59,76,93,184,216],"both":[61,118],"those":[63],"neural":[67,135],"architectures.":[68],"However,":[69],"gain":[72],"from":[73,98],"these":[74],"is":[77],"achieved":[78],"under":[79],"different":[80],"spaces":[82],"training":[84],"setups.":[85],"This":[86,202],"makes":[87],"overall":[89],"incomparable":[94],"improvement":[97],"sub-module":[100],"model":[104],"unclear.":[105],"In":[106],"this":[107],"paper,":[108],"we":[109],"propose":[110],"NATS-Bench,":[111],"unified":[113],"benchmark":[114,155],"on":[115,146,185,212],"size,":[121],"(almost)":[123],"any":[124],"up-to-date":[125],"NAS":[126,183,215],"algorithm.":[127],"NATS-Bench":[128,177],"includes":[129],"space":[132],"15,625":[134],"cell":[136],"candidates":[137,166],"32,768":[142],"three":[147],"datasets.":[148],"We":[149,171],"analyze":[150],"validity":[152],"our":[154],"terms":[157],"various":[159],"criteria":[160],"comparison":[163],"all":[165],"space.":[170],"also":[172],"show":[173],"versatility":[175],"by":[178],"benchmarking":[179],"13":[180],"recent":[181],"state-of-the-art":[182],"it.":[186],"All":[187,225],"logs":[188],"diagnostic":[190],"information":[191],"trained":[192],"using":[193],"same":[195],"setup":[196],"each":[198],"candidate":[199],"are":[200,227],"provided.":[201],"facilitates":[203],"much":[205],"larger":[206],"researchers":[209],"focus":[211],"developing":[213],"better":[214],"more":[219],"comparable":[220],"computationally":[222],"effective":[223],"environment.":[224],"codes":[226],"publicly":[228],"available":[229],"at:":[230],"https://xuanyidong.com/assets/projects/NATS-Bench.":[231]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":32},{"year":2024,"cited_by_count":32},{"year":2023,"cited_by_count":35},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":34},{"year":2020,"cited_by_count":2}],"updated_date":"2026-04-05T17:49:38.594831","created_date":"2025-10-10T00:00:00"}