{"id":"https://openalex.org/W3043413334","doi":"https://doi.org/10.14778/3446095.3446100","title":"Analyzing and mitigating data stalls in DNN training","display_name":"Analyzing and mitigating data stalls in DNN training","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W3043413334","doi":"https://doi.org/10.14778/3446095.3446100","mag":"3043413334"},"language":"en","primary_location":{"id":"doi:10.14778/3446095.3446100","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3446095.3446100","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2007.06775","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5005136390","display_name":"Jayashree Mohan","orcid":"https://orcid.org/0009-0005-5260-3203"},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jayashree Mohan","raw_affiliation_strings":["University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5011102459","display_name":"Amar Phanishayee","orcid":"https://orcid.org/0009-0001-2777-1118"},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Amar Phanishayee","raw_affiliation_strings":["Microsoft Research","Microsoft Research#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"Microsoft Research#TAB#","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014304030","display_name":"Ashish Raniwala","orcid":null},"institutions":[{"id":"https://openalex.org/I4210164937","display_name":"Microsoft Research (United Kingdom)","ror":"https://ror.org/05k87vq12","country_code":"GB","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210164937"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Ashish Raniwala","raw_affiliation_strings":["Microsoft","(Microsoft)"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft","institution_ids":["https://openalex.org/I4210164937"]},{"raw_affiliation_string":"(Microsoft)","institution_ids":["https://openalex.org/I4210164937"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010361101","display_name":"Vijay Chidambaram","orcid":"https://orcid.org/0000-0001-7985-6087"},"institutions":[{"id":"https://openalex.org/I74786994","display_name":"Kitware (United States)","ror":"https://ror.org/02s2acn37","country_code":"US","type":"company","lineage":["https://openalex.org/I74786994"]},{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Vijay Chidambaram","raw_affiliation_strings":["University of Texas at Austin & VMWare Research","University of Texas at Austin and VMware Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin & VMWare Research","institution_ids":["https://openalex.org/I74786994"]},{"raw_affiliation_string":"University of Texas at Austin and VMware Research","institution_ids":["https://openalex.org/I74786994","https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.7247,"has_fulltext":true,"cited_by_count":10,"citation_normalized_percentile":{"value":0.67417229,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":93,"max":97},"biblio":{"volume":"14","issue":"5","first_page":"771","last_page":"784"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9958999752998352,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8505549430847168},{"id":"https://openalex.org/keywords/preprocessor","display_name":"Preprocessor","score":0.6318463683128357},{"id":"https://openalex.org/keywords/pipeline","display_name":"Pipeline (software)","score":0.5927066802978516},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5527674555778503},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.5365880131721497},{"id":"https://openalex.org/keywords/raw-data","display_name":"Raw data","score":0.49078497290611267},{"id":"https://openalex.org/keywords/data-pre-processing","display_name":"Data pre-processing","score":0.48780038952827454},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4346398711204529},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.42060959339141846},{"id":"https://openalex.org/keywords/deep-neural-networks","display_name":"Deep neural networks","score":0.4112512469291687},{"id":"https://openalex.org/keywords/data-mining","display_name":"Data mining","score":0.3904111385345459},{"id":"https://openalex.org/keywords/real-time-computing","display_name":"Real-time computing","score":0.3448852300643921},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.21365967392921448}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8505549430847168},{"id":"https://openalex.org/C34736171","wikidata":"https://www.wikidata.org/wiki/Q918333","display_name":"Preprocessor","level":2,"score":0.6318463683128357},{"id":"https://openalex.org/C43521106","wikidata":"https://www.wikidata.org/wiki/Q2165493","display_name":"Pipeline (software)","level":2,"score":0.5927066802978516},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5527674555778503},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.5365880131721497},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.49078497290611267},{"id":"https://openalex.org/C10551718","wikidata":"https://www.wikidata.org/wiki/Q5227332","display_name":"Data pre-processing","level":2,"score":0.48780038952827454},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4346398711204529},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42060959339141846},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.4112512469291687},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.3904111385345459},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.3448852300643921},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.21365967392921448},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.14778/3446095.3446100","is_oa":false,"landing_page_url":"https://doi.org/10.14778/3446095.3446100","pdf_url":null,"source":{"id":"https://openalex.org/S4210226185","display_name":"Proceedings of the VLDB Endowment","issn_l":"2150-8097","issn":["2150-8097"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the VLDB Endowment","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2007.06775","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.06775","pdf_url":"https://arxiv.org/pdf/2007.06775","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"mag:3043413334","is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/2007.06775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2007.06775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2007.06775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2007.06775","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2007.06775","pdf_url":"https://arxiv.org/pdf/2007.06775","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[{"score":0.4300000071525574,"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3043413334.pdf","grobid_xml":"https://content.openalex.org/works/W3043413334.grobid-xml"},"referenced_works_count":76,"referenced_works":["https://openalex.org/W1442374986","https://openalex.org/W1566289585","https://openalex.org/W1598866093","https://openalex.org/W1686810756","https://openalex.org/W2008633193","https://openalex.org/W2097998348","https://openalex.org/W2101281312","https://openalex.org/W2108598243","https://openalex.org/W2117539524","https://openalex.org/W2120754641","https://openalex.org/W2133156997","https://openalex.org/W2139501017","https://openalex.org/W2143612262","https://openalex.org/W2155651153","https://openalex.org/W2163605009","https://openalex.org/W2164433331","https://openalex.org/W2168231600","https://openalex.org/W2193145675","https://openalex.org/W2194775991","https://openalex.org/W2272300165","https://openalex.org/W2279098554","https://openalex.org/W2338908902","https://openalex.org/W2524365899","https://openalex.org/W2525778437","https://openalex.org/W2529337537","https://openalex.org/W2580221632","https://openalex.org/W2583383421","https://openalex.org/W2606722458","https://openalex.org/W2617242334","https://openalex.org/W2622263826","https://openalex.org/W2732547613","https://openalex.org/W2766164908","https://openalex.org/W2770298516","https://openalex.org/W2775795276","https://openalex.org/W2786320458","https://openalex.org/W2788079077","https://openalex.org/W2804032941","https://openalex.org/W2883265831","https://openalex.org/W2883283076","https://openalex.org/W2884700152","https://openalex.org/W2888655500","https://openalex.org/W2899071864","https://openalex.org/W2899505139","https://openalex.org/W2900042580","https://openalex.org/W2910100551","https://openalex.org/W2910132948","https://openalex.org/W2919594608","https://openalex.org/W2948438818","https://openalex.org/W2949433733","https://openalex.org/W2962758826","https://openalex.org/W2963125010","https://openalex.org/W2963163009","https://openalex.org/W2963341956","https://openalex.org/W2963403751","https://openalex.org/W2963803379","https://openalex.org/W2963815651","https://openalex.org/W2963933775","https://openalex.org/W2964174152","https://openalex.org/W2964284859","https://openalex.org/W2969388332","https://openalex.org/W2981758446","https://openalex.org/W2991040477","https://openalex.org/W2994779554","https://openalex.org/W3005665451","https://openalex.org/W3010637564","https://openalex.org/W3011145936","https://openalex.org/W3022298203","https://openalex.org/W3037338728","https://openalex.org/W3037512697","https://openalex.org/W3037875189","https://openalex.org/W3046648263","https://openalex.org/W3081844508","https://openalex.org/W3106250896","https://openalex.org/W3118608800","https://openalex.org/W3176045042","https://openalex.org/W4241850174"],"related_works":["https://openalex.org/W2917673503","https://openalex.org/W2884702064","https://openalex.org/W3137382871","https://openalex.org/W2908737781","https://openalex.org/W3006123172","https://openalex.org/W3018825982","https://openalex.org/W2812702525","https://openalex.org/W2967854646","https://openalex.org/W2808215910","https://openalex.org/W2909939200","https://openalex.org/W3130104841","https://openalex.org/W3166015079","https://openalex.org/W106762293","https://openalex.org/W2260621657","https://openalex.org/W2904654489","https://openalex.org/W2532379532","https://openalex.org/W2969378954","https://openalex.org/W2160741692","https://openalex.org/W2981186140","https://openalex.org/W2760592707"],"abstract_inverted_index":{"Training":[0],"Deep":[1,75],"Neural":[2,76],"Networks":[3,77],"(DNNs)":[4],"is":[5,136,233],"resource-intensive":[6],"and":[7,36,73,93,151,168,187,213],"time-consuming.":[8],"While":[9],"prior":[10],"research":[11],"has":[12,42],"explored":[13],"many":[14,131],"different":[15,88],"ways":[16],"of":[17,24,59,69,103,106,120,208,223],"reducing":[18],"DNN":[19,133,209,230],"training":[20,67,134,231],"time,":[21],"the":[22,49,55,61,66,101,180,224],"impact":[23],"input":[25,62],"data":[26,32,38,63,83,139,147,162,174,201,227],"pipeline":[27,64],",":[28],"i.e.,":[29],"fetching":[30],"raw":[31],"items":[33],"from":[34,182],"storage":[35,109],"performing":[37],"pre-processing":[39],"in":[40,130,194],"memory,":[41,104],"been":[43],"relatively":[44],"unexplored.":[45],"This":[46],"paper":[47],"makes":[48],"following":[50],"contributions:":[51],"(1)":[52],"We":[53,85,127,154],"present":[54],"first":[56],"comprehensive":[57],"analysis":[58,172],"how":[60],"affects":[65],"time":[68,135,141,143,232],"widely-used":[70],"computer":[71],"vision":[72],"audio":[74],"(DNNs),":[78],"that":[79,116,129,217],"typically":[80],"involve":[81],"complex":[82],"pre-processing.":[84],"analyze":[86],"nine":[87],"models":[89],"across":[90],"three":[91,189],"tasks":[92],"four":[94],"datasets":[95],"while":[96],"varying":[97],"factors":[98],"such":[99],"as":[100,237,239],"amount":[102],"number":[105],"CPU":[107],"threads,":[108],"device,":[110],"GPU":[111],"generation":[112],"etc":[113],"on":[114,173,179,205,241],"servers":[115],"are":[117],"a":[118,121,156,165,195,206,242],"part":[119],"large":[122],"production":[123],"cluster":[124],"at":[125],"Microsoft.":[126],"find":[128],"cases,":[132],"dominated":[137],"by":[138],"stall":[140],":":[142],"spent":[144],"waiting":[145],"for":[146],"to":[148,159,199],"be":[149],"fetched":[150],"pre-processed.":[152],"(2)":[153],"build":[155],"tool,":[157],"DS-Analyzer":[158],"precisely":[160],"measure":[161],"stalls":[163],"using":[164],"differential":[166],"technique,":[167],"perform":[169],"predictive":[170],"what-if":[171],"stalls.":[175,202],"(3)":[176],"Finally,":[177],"based":[178],"insights":[181],"our":[183],"analysis,":[184],"we":[185],"design":[186],"implement":[188],"simple":[190],"but":[191],"effective":[192],"techniques":[193],"data-loading":[196],"library,":[197,229],"CoorDL,":[198],"mitigate":[200],"Our":[203],"experiments":[204],"range":[207],"tasks,":[210],"models,":[211],"datasets,":[212],"hardware":[214],"configs":[215],"show":[216],"when":[218],"PyTorch":[219],"uses":[220],"CoorDL":[221],"instead":[222],"state-of-the-art":[225],"DALI":[226],"loading":[228],"reduced":[234],"significantly":[235],"(by":[236],"much":[238],"5X":[240],"single":[243],"server).":[244]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}