{"id":"https://openalex.org/W4387717737","doi":"https://doi.org/10.1109/mm.2023.3324261","title":"NetDistiller: Empowering Tiny Deep Learning via In Situ Distillation","display_name":"NetDistiller: Empowering Tiny Deep Learning via In Situ Distillation","publication_year":2023,"publication_date":"2023-10-17","ids":{"openalex":"https://openalex.org/W4387717737","doi":"https://doi.org/10.1109/mm.2023.3324261"},"language":"en","primary_location":{"id":"doi:10.1109/mm.2023.3324261","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2023.3324261","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102728141","display_name":"Shunyao Zhang","orcid":"https://orcid.org/0000-0002-6227-6905"},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shunyao Zhang","raw_affiliation_strings":["Rice University, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-6227-6905","affiliations":[{"raw_affiliation_string":"Rice University, Houston, TX, USA","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061572132","display_name":"Yonggan Fu","orcid":"https://orcid.org/0000-0002-7483-2921"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yonggan Fu","raw_affiliation_strings":["Georgia Institute of Technology, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0002-7483-2921","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076738815","display_name":"Shang Wu","orcid":"https://orcid.org/0000-0001-6467-7166"},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Shang Wu","raw_affiliation_strings":["Rice University, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0001-6467-7166","affiliations":[{"raw_affiliation_string":"Rice University, Houston, TX, USA","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065225892","display_name":"Jyotikrishna Dass","orcid":"https://orcid.org/0000-0002-7552-8753"},"institutions":[{"id":"https://openalex.org/I74775410","display_name":"Rice University","ror":"https://ror.org/008zs3103","country_code":"US","type":"education","lineage":["https://openalex.org/I74775410"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jyotikrishna Dass","raw_affiliation_strings":["Rice University, Houston, TX, USA"],"raw_orcid":"https://orcid.org/0000-0002-7552-8753","affiliations":[{"raw_affiliation_string":"Rice University, Houston, TX, USA","institution_ids":["https://openalex.org/I74775410"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080793133","display_name":"Haoran You","orcid":null},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Haoran You","raw_affiliation_strings":["Georgia Institute of Technology School of Computer Science, Machine Learning Center, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0002-2873-2153","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology School of Computer Science, Machine Learning Center, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5019582323","display_name":"Yingyan Lin","orcid":"https://orcid.org/0000-0001-5946-203X"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yingyan Lin","raw_affiliation_strings":["Georgia Institute of Technology School of Computer Science, Machine Learning Center, Atlanta, GA, USA"],"raw_orcid":"https://orcid.org/0000-0001-5946-203X","affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology School of Computer Science, Machine Learning Center, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1064,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.39476695,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":94},"biblio":{"volume":"43","issue":"6","first_page":"84","last_page":"92"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9940999746322632,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8796252608299255},{"id":"https://openalex.org/keywords/overfitting","display_name":"Overfitting","score":0.7463794946670532},{"id":"https://openalex.org/keywords/boosting","display_name":"Boosting (machine learning)","score":0.6553926467895508},{"id":"https://openalex.org/keywords/edge-device","display_name":"Edge device","score":0.5919787287712097},{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.5263603925704956},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.518462598323822},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5165254473686218},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.5149675011634827},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4707285463809967},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4461553692817688},{"id":"https://openalex.org/keywords/bandwidth","display_name":"Bandwidth (computing)","score":0.41769057512283325},{"id":"https://openalex.org/keywords/distributed-computing","display_name":"Distributed computing","score":0.39345648884773254},{"id":"https://openalex.org/keywords/computer-engineering","display_name":"Computer engineering","score":0.38890016078948975},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.12459984421730042},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.1207859218120575},{"id":"https://openalex.org/keywords/telecommunications","display_name":"Telecommunications","score":0.1019880473613739}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8796252608299255},{"id":"https://openalex.org/C22019652","wikidata":"https://www.wikidata.org/wiki/Q331309","display_name":"Overfitting","level":3,"score":0.7463794946670532},{"id":"https://openalex.org/C46686674","wikidata":"https://www.wikidata.org/wiki/Q466303","display_name":"Boosting (machine learning)","level":2,"score":0.6553926467895508},{"id":"https://openalex.org/C138236772","wikidata":"https://www.wikidata.org/wiki/Q25098575","display_name":"Edge device","level":3,"score":0.5919787287712097},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.5263603925704956},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.518462598323822},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5165254473686218},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.5149675011634827},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4707285463809967},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4461553692817688},{"id":"https://openalex.org/C2776257435","wikidata":"https://www.wikidata.org/wiki/Q1576430","display_name":"Bandwidth (computing)","level":2,"score":0.41769057512283325},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.39345648884773254},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.38890016078948975},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.12459984421730042},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.1207859218120575},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.1019880473613739},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C79974875","wikidata":"https://www.wikidata.org/wiki/Q483639","display_name":"Cloud computing","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/mm.2023.3324261","is_oa":false,"landing_page_url":"https://doi.org/10.1109/mm.2023.3324261","pdf_url":null,"source":{"id":"https://openalex.org/S59697426","display_name":"IEEE Micro","issn_l":"0272-1732","issn":["0272-1732","1937-4143"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Micro","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/4","score":0.7900000214576721,"display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G6549010072","display_name":"SHF: Medium:DILSE: Codesigning Decentralized Incremental Learning System via Streaming Data Summarization on Edge","funder_award_id":"2211815","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":15,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W2967733054","https://openalex.org/W3004127093","https://openalex.org/W3044604993","https://openalex.org/W4221159017","https://openalex.org/W4297775537","https://openalex.org/W6638523607","https://openalex.org/W6679909955","https://openalex.org/W6737664043","https://openalex.org/W6762718338","https://openalex.org/W6771876938","https://openalex.org/W6780827055","https://openalex.org/W6791110730","https://openalex.org/W6803315452","https://openalex.org/W6809824710"],"related_works":["https://openalex.org/W2393964553","https://openalex.org/W4390238976","https://openalex.org/W3176282186","https://openalex.org/W2973622361","https://openalex.org/W4387489555","https://openalex.org/W3185576471","https://openalex.org/W4288024917","https://openalex.org/W4293053895","https://openalex.org/W2983364019","https://openalex.org/W2998183476"],"abstract_inverted_index":{"Boosting":[0],"the":[1,16,49,66,71,74,82,92,103,106],"task":[2],"accuracy":[3,51,121],"of":[4,18,31,52,59,68,70,105],"tiny":[5],"neural":[6],"networks":[7],"(TNNs)":[8],"has":[9],"become":[10],"a":[11,43,60],"fundamental":[12],"challenge":[13],"for":[14],"enabling":[15],"deployment":[17],"TNNs":[19,53],"on":[20],"edge":[21],"devices,":[22],"which":[23],"are":[24],"constrained":[25],"by":[26,54,64],"strict":[27],"limitations":[28],"in":[29,117],"terms":[30],"memory,":[32],"computation,":[33],"bandwidth,":[34],"and":[35,97],"power":[36],"supply.":[37],"To":[38],"this":[39],"end,":[40],"we":[41],"propose":[42],"framework":[44],"called":[45],"NetDistiller":[46],"to":[47,90,101],"boost":[48],"achievable":[50,120],"treating":[55],"them":[56,96],"as":[57],"subnetworks":[58],"weight-sharing":[61,83],"teacher":[62,84,107],"constructed":[63],"expanding":[65],"number":[67],"channels":[69],"TNN.":[72],"Specifically,":[73],"target":[75],"TNN":[76],"model":[77,85],"is":[78,127],"jointly":[79],"trained":[80],"with":[81],"via":[86],"1)":[87],"gradient":[88,93],"surgery":[89],"tackle":[91],"conflicts":[94],"between":[95],"2)":[98],"uncertainty-aware":[99],"distillation":[100],"mitigate":[102],"overfitting":[104],"model.":[108],"Extensive":[109],"experiments":[110],"across":[111],"diverse":[112],"tasks":[113],"validate":[114],"NetDistiller\u2019s":[115],"effectiveness":[116],"boosting":[118],"TNNs\u2019":[119],"over":[122],"state-of-the-art":[123],"methods.":[124],"Our":[125],"code":[126],"available":[128],"at":[129],"https://github.com/GATECH-EIC/NetDistiller.":[130]},"counts_by_year":[{"year":2023,"cited_by_count":1}],"updated_date":"2026-03-27T05:58:40.876381","created_date":"2025-10-10T00:00:00"}