{"id":"https://openalex.org/W2805214778","doi":"https://doi.org/10.1145/3352020.3352024","title":"Analysis of DAWNBench, a Time-to-Accuracy Machine Learning Performance Benchmark","display_name":"Analysis of DAWNBench, a Time-to-Accuracy Machine Learning Performance Benchmark","publication_year":2019,"publication_date":"2019-07-25","ids":{"openalex":"https://openalex.org/W2805214778","doi":"https://doi.org/10.1145/3352020.3352024","mag":"2805214778"},"language":"en","primary_location":{"id":"doi:10.1145/3352020.3352024","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3352020.3352024","pdf_url":null,"source":{"id":"https://openalex.org/S50071195","display_name":"ACM SIGOPS Operating Systems Review","issn_l":"0163-5980","issn":["0163-5980","1943-586X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGOPS Operating Systems Review","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1806.01427","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039941658","display_name":"Cody Coleman","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Cody Coleman","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5072348548","display_name":"Daniel Kang","orcid":"https://orcid.org/0000-0001-9860-9938"},"institutions":[{"id":"https://openalex.org/I4210109743","display_name":"DAWN Center for Independent Living","ror":"https://ror.org/01dwm7q20","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210109743"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Daniel Kang","raw_affiliation_strings":["Stanford DAWN, Standford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Standford, CA, USA","institution_ids":["https://openalex.org/I4210109743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079514101","display_name":"Deepak Narayanan","orcid":"https://orcid.org/0000-0002-3020-2848"},"institutions":[{"id":"https://openalex.org/I4210109743","display_name":"DAWN Center for Independent Living","ror":"https://ror.org/01dwm7q20","country_code":"US","type":"nonprofit","lineage":["https://openalex.org/I4210109743"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Deepak Narayanan","raw_affiliation_strings":["Stanford DAWN, Standford, CA, USA"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Standford, CA, USA","institution_ids":["https://openalex.org/I4210109743"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008975166","display_name":"Luigi Nardi","orcid":"https://orcid.org/0000-0002-4601-2264"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luigi Nardi","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064644506","display_name":"Tian Zhao","orcid":"https://orcid.org/0000-0001-5372-0835"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tian Zhao","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100409994","display_name":"Jian Zhang","orcid":"https://orcid.org/0000-0002-7240-3541"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jian Zhang","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5002538469","display_name":"Peter Bailis","orcid":"https://orcid.org/0000-0003-1166-7823"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peter Bailis","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023857198","display_name":"Kunle Olukotun","orcid":"https://orcid.org/0000-0002-8779-0636"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kunle Olukotun","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103148807","display_name":"Chris R\u00e9","orcid":"https://orcid.org/0000-0002-4109-0108"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chris R\u00e9","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5005554337","display_name":"Matei Zaharia","orcid":"https://orcid.org/0000-0002-7547-7204"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Matei Zaharia","raw_affiliation_strings":["Stanford DAWN, Stanford, CA, USA","Stanford DAWN, Stanford, CA, USA#TAB#"],"affiliations":[{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA","institution_ids":[]},{"raw_affiliation_string":"Stanford DAWN, Stanford, CA, USA#TAB#","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":10,"corresponding_author_ids":["https://openalex.org/A5039941658"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.6131,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.71218996,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"53","issue":"1","first_page":"14","last_page":"25"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9901000261306763,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.9275751113891602},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8213984966278076},{"id":"https://openalex.org/keywords/metric","display_name":"Metric (unit)","score":0.7530529499053955},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5115540623664856},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45470625162124634},{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.4392300546169281},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.4147791564464569},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.41023150086402893},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.08247655630111694}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.9275751113891602},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8213984966278076},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.7530529499053955},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5115540623664856},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45470625162124634},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.4392300546169281},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.4147791564464569},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.41023150086402893},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.08247655630111694},{"id":"https://openalex.org/C44870925","wikidata":"https://www.wikidata.org/wiki/Q37547","display_name":"Astrophysics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1145/3352020.3352024","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3352020.3352024","pdf_url":null,"source":{"id":"https://openalex.org/S50071195","display_name":"ACM SIGOPS Operating Systems Review","issn_l":"0163-5980","issn":["0163-5980","1943-586X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGOPS Operating Systems Review","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1806.01427","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1806.01427","pdf_url":"https://arxiv.org/pdf/1806.01427","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.1806.01427","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1806.01427","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"},{"id":"mag:2805214778","is_oa":false,"landing_page_url":null,"pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:1806.01427","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1806.01427","pdf_url":"https://arxiv.org/pdf/1806.01427","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2805214778.pdf","grobid_xml":"https://content.openalex.org/works/W2805214778.grobid-xml"},"referenced_works_count":68,"referenced_works":["https://openalex.org/W104184427","https://openalex.org/W1442374986","https://openalex.org/W1491468723","https://openalex.org/W1522301498","https://openalex.org/W1665214252","https://openalex.org/W1667652561","https://openalex.org/W1788418780","https://openalex.org/W1994616650","https://openalex.org/W2002555321","https://openalex.org/W2083842231","https://openalex.org/W2108598243","https://openalex.org/W2125901106","https://openalex.org/W2138243089","https://openalex.org/W2155893237","https://openalex.org/W2168231600","https://openalex.org/W2186615578","https://openalex.org/W2259472270","https://openalex.org/W2279098554","https://openalex.org/W2332506150","https://openalex.org/W2402144811","https://openalex.org/W2514858228","https://openalex.org/W2515080096","https://openalex.org/W2523435939","https://openalex.org/W2606722458","https://openalex.org/W2619516334","https://openalex.org/W2622263826","https://openalex.org/W2624299682","https://openalex.org/W2647836899","https://openalex.org/W2734663976","https://openalex.org/W2735224642","https://openalex.org/W2763421725","https://openalex.org/W2766164908","https://openalex.org/W2766527293","https://openalex.org/W2769856846","https://openalex.org/W2774000609","https://openalex.org/W2785430118","https://openalex.org/W2787998955","https://openalex.org/W2790501674","https://openalex.org/W2791110811","https://openalex.org/W2799042347","https://openalex.org/W2804032941","https://openalex.org/W2806021783","https://openalex.org/W2808118451","https://openalex.org/W2884700152","https://openalex.org/W2894175714","https://openalex.org/W2899771611","https://openalex.org/W2903697572","https://openalex.org/W2949117887","https://openalex.org/W2951714314","https://openalex.org/W2951781666","https://openalex.org/W2953212265","https://openalex.org/W2962747323","https://openalex.org/W2962843773","https://openalex.org/W2963208657","https://openalex.org/W2963372104","https://openalex.org/W2963674387","https://openalex.org/W2963792515","https://openalex.org/W2963803379","https://openalex.org/W2964121744","https://openalex.org/W2997613220","https://openalex.org/W2998553639","https://openalex.org/W3037847693","https://openalex.org/W3101543398","https://openalex.org/W4211178602","https://openalex.org/W6648914341","https://openalex.org/W6682889407","https://openalex.org/W6730267373","https://openalex.org/W6756040250"],"related_works":["https://openalex.org/W2966048283","https://openalex.org/W2194775991","https://openalex.org/W2108598243","https://openalex.org/W3195958055","https://openalex.org/W1762450390","https://openalex.org/W1583189038","https://openalex.org/W2965980968","https://openalex.org/W1592851408","https://openalex.org/W130698394","https://openalex.org/W2889225139","https://openalex.org/W2408928403","https://openalex.org/W1966670013","https://openalex.org/W2243363624","https://openalex.org/W2018376578","https://openalex.org/W160589420","https://openalex.org/W3197139525","https://openalex.org/W2562785202","https://openalex.org/W2152436418","https://openalex.org/W3195140297","https://openalex.org/W3139128787"],"abstract_inverted_index":{"Researchers":[0],"have":[1],"proposed":[2],"hardware,":[3],"software,":[4],"and":[5,43,146],"algorithmic":[6],"optimizations":[7,20],"to":[8,54,68,89,119,169,210],"improve":[9],"the":[10,22,34,37,46,107,121,133,211],"computational":[11],"performance":[12],"of":[13,18,36,57,123,144,199],"deep":[14],"learning.":[15],"While":[16],"some":[17],"these":[19,63,71],"perform":[21],"same":[23],"operations":[24],"faster":[25],"(e.g.,":[26,40],"increasing":[27],"GPU":[28],"clock":[29],"speed),":[30],"many":[31],"others":[32],"modify":[33],"semantics":[35],"training":[38,87],"procedure":[39],"reduced":[41],"precision),":[42],"can":[44,194],"impact":[45],"final":[47],"model's":[48],"accuracy":[49,92],"on":[50,85,93,202],"unseen":[51,95],"data.":[52],"Due":[53],"a":[55,81,126,141],"lack":[56],"standard":[58,160],"evaluation":[59],"criteria":[60],"that":[61,138,147,191],"considers":[62],"trade-offs,":[64],"it":[65],"is":[66],"difficult":[67],"directly":[69],"compare":[70],"optimizations.":[72],"To":[73],"address":[74],"this":[75,103],"problem,":[76],"we":[77,105,177,189],"recently":[78],"introduced":[79],"DAWNBENCH,":[80,110],"benchmark":[82],"competition":[83],"focused":[84],"end-to-end":[86],"time":[88,201],"achieve":[90],"near-state-of-the-art":[91],"an":[94],"dataset-a":[96],"combined":[97],"metric":[98,127],"called":[99],"time-to-accuracy":[100],"(TTA).":[101],"In":[102],"work,":[104],"analyze":[106],"entries":[108,166,193,209],"from":[109,115],"which":[111],"received":[112],"optimized":[113,149],"submissions":[114],"multiple":[116],"industrial":[117],"groups,":[118],"investigate":[120],"behavior":[122],"TTA":[124,139,151],"as":[125,128,130,154,156,185],"well":[129,155],"trends":[131],"in":[132,173],"best-performing":[134],"entries.":[135],"We":[136,204],"show":[137,205],"has":[140],"low":[142],"coefficient":[143],"variation":[145],"models":[148,172],"for":[150],"generalize":[152],"nearly":[153],"those":[157],"trained":[158],"using":[159],"methods.":[161],"Additionally,":[162],"even":[163],"though":[164],"DAWNBENCH":[165],"were":[167],"able":[168],"train":[170],"ImageNet":[171],"under":[174],"3":[175],"minutes,":[176],"find":[178,190],"they":[179],"still":[180],"underutilize":[181],"hardware":[182],"capabilities":[183],"such":[184],"Tensor":[186],"Cores.":[187],"Furthermore,":[188],"distributed":[192],"spend":[195],"more":[196],"than":[197],"half":[198],"their":[200],"communication.":[203],"similar":[206],"findings":[207],"with":[208],"MLPERF":[212],"v0.5":[213],"benchmark.":[214]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":2},{"year":2018,"cited_by_count":2}],"updated_date":"2026-03-21T08:13:44.787528","created_date":"2022-10-03T00:00:00"}